chore: remove ~50k lines of unreachable dead code (#8913)
* chore: remove unreachable dead code across the codebase Remove ~50,000 lines of unreachable code identified by static analysis. Major removals: - weed/filer/redis_lua: entire unused Redis Lua filer store implementation - weed/wdclient/net2, resource_pool: unused connection/resource pool packages - weed/plugin/worker/lifecycle: unused lifecycle plugin worker - weed/s3api: unused S3 policy templates, presigned URL IAM, streaming copy, multipart IAM, key rotation, and various SSE helper functions - weed/mq/kafka: unused partition mapping, compression, schema, and protocol functions - weed/mq/offset: unused SQL storage and migration code - weed/worker: unused registry, task, and monitoring functions - weed/query: unused SQL engine, parquet scanner, and type functions - weed/shell: unused EC proportional rebalance functions - weed/storage/erasure_coding/distribution: unused distribution analysis functions - Individual unreachable functions removed from 150+ files across admin, credential, filer, iam, kms, mount, mq, operation, pb, s3api, server, shell, storage, topology, and util packages * fix(s3): reset shared memory store in IAM test to prevent flaky failure TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey was flaky because the MemoryStore credential backend is a singleton registered via init(). Earlier tests that create anonymous identities pollute the shared store, causing LookupAnonymous() to unexpectedly return true. Fix by calling Reset() on the memory store before the test runs. * style: run gofmt on changed files * fix: restore KMS functions used by integration tests * fix(plugin): prevent panic on send to closed worker session channel The Plugin.sendToWorker method could panic with "send on closed channel" when a worker disconnected while a message was being sent. The race was between streamSession.close() closing the outgoing channel and sendToWorker writing to it concurrently. Add a done channel to streamSession that is closed before the outgoing channel, and check it in sendToWorker's select to safely detect closed sessions without panicking.
This commit is contained in:
Chris Lu
@@ -449,58 +449,6 @@ func hasEligibleCompaction(
|
||||
return len(bins) > 0, nil
|
||||
}
|
||||
|
||||
func countDataManifestsForRewrite(
|
||||
ctx context.Context,
|
||||
filerClient filer_pb.SeaweedFilerClient,
|
||||
bucketName, tablePath string,
|
||||
manifests []iceberg.ManifestFile,
|
||||
meta table.Metadata,
|
||||
predicate *partitionPredicate,
|
||||
) (int64, error) {
|
||||
if predicate == nil {
|
||||
return countDataManifests(manifests), nil
|
||||
}
|
||||
|
||||
specsByID := specByID(meta)
|
||||
|
||||
var count int64
|
||||
for _, mf := range manifests {
|
||||
if mf.ManifestContent() != iceberg.ManifestContentData {
|
||||
continue
|
||||
}
|
||||
manifestData, err := loadFileByIcebergPath(ctx, filerClient, bucketName, tablePath, mf.FilePath())
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("read manifest %s: %w", mf.FilePath(), err)
|
||||
}
|
||||
entries, err := iceberg.ReadManifest(mf, bytes.NewReader(manifestData), true)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("parse manifest %s: %w", mf.FilePath(), err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
continue
|
||||
}
|
||||
spec, ok := specsByID[int(mf.PartitionSpecID())]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
allMatch := len(entries) > 0
|
||||
for _, entry := range entries {
|
||||
match, err := predicate.Matches(spec, entry.DataFile().Partition())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !match {
|
||||
allMatch = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allMatch {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func compactionMinInputFiles(minInputFiles int64) (int, error) {
|
||||
// Ensure the configured value is positive and fits into the platform's int type
|
||||
if minInputFiles <= 0 {
|
||||
|
||||
@@ -137,26 +137,6 @@ func mergePlanningIndexSections(index, existing *planningIndex) *planningIndex {
|
||||
return index
|
||||
}
|
||||
|
||||
func buildPlanningIndex(
|
||||
ctx context.Context,
|
||||
filerClient filer_pb.SeaweedFilerClient,
|
||||
bucketName, tablePath string,
|
||||
meta table.Metadata,
|
||||
config Config,
|
||||
ops []string,
|
||||
) (*planningIndex, error) {
|
||||
currentSnap := meta.CurrentSnapshot()
|
||||
if currentSnap == nil || currentSnap.ManifestList == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
manifests, err := loadCurrentManifests(ctx, filerClient, bucketName, tablePath, meta)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildPlanningIndexFromManifests(ctx, filerClient, bucketName, tablePath, meta, config, ops, manifests)
|
||||
}
|
||||
|
||||
func buildPlanningIndexFromManifests(
|
||||
ctx context.Context,
|
||||
filerClient filer_pb.SeaweedFilerClient,
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
const (
|
||||
jobType = "s3_lifecycle"
|
||||
|
||||
defaultBatchSize = 1000
|
||||
defaultMaxDeletesPerBucket = 10000
|
||||
defaultDryRun = false
|
||||
defaultDeleteMarkerCleanup = true
|
||||
defaultAbortMPUDaysDefault = 7
|
||||
|
||||
MetricObjectsExpired = "objects_expired"
|
||||
MetricObjectsScanned = "objects_scanned"
|
||||
MetricBucketsScanned = "buckets_scanned"
|
||||
MetricBucketsWithRules = "buckets_with_rules"
|
||||
MetricDeleteMarkersClean = "delete_markers_cleaned"
|
||||
MetricMPUAborted = "mpu_aborted"
|
||||
MetricErrors = "errors"
|
||||
MetricDurationMs = "duration_ms"
|
||||
)
|
||||
|
||||
// Config holds parsed worker config values for lifecycle management.
|
||||
type Config struct {
|
||||
BatchSize int64
|
||||
MaxDeletesPerBucket int64
|
||||
DryRun bool
|
||||
DeleteMarkerCleanup bool
|
||||
AbortMPUDays int64
|
||||
}
|
||||
|
||||
// ParseConfig extracts a lifecycle Config from plugin config values.
|
||||
func ParseConfig(values map[string]*plugin_pb.ConfigValue) Config {
|
||||
cfg := Config{
|
||||
BatchSize: readInt64Config(values, "batch_size", defaultBatchSize),
|
||||
MaxDeletesPerBucket: readInt64Config(values, "max_deletes_per_bucket", defaultMaxDeletesPerBucket),
|
||||
DryRun: readBoolConfig(values, "dry_run", defaultDryRun),
|
||||
DeleteMarkerCleanup: readBoolConfig(values, "delete_marker_cleanup", defaultDeleteMarkerCleanup),
|
||||
AbortMPUDays: readInt64Config(values, "abort_mpu_days", defaultAbortMPUDaysDefault),
|
||||
}
|
||||
|
||||
if cfg.BatchSize <= 0 {
|
||||
cfg.BatchSize = defaultBatchSize
|
||||
}
|
||||
if cfg.MaxDeletesPerBucket <= 0 {
|
||||
cfg.MaxDeletesPerBucket = defaultMaxDeletesPerBucket
|
||||
}
|
||||
if cfg.AbortMPUDays < 0 {
|
||||
cfg.AbortMPUDays = defaultAbortMPUDaysDefault
|
||||
}
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
func readStringConfig(values map[string]*plugin_pb.ConfigValue, field string, fallback string) string {
|
||||
if values == nil {
|
||||
return fallback
|
||||
}
|
||||
value := values[field]
|
||||
if value == nil {
|
||||
return fallback
|
||||
}
|
||||
switch kind := value.Kind.(type) {
|
||||
case *plugin_pb.ConfigValue_StringValue:
|
||||
return kind.StringValue
|
||||
case *plugin_pb.ConfigValue_Int64Value:
|
||||
return strconv.FormatInt(kind.Int64Value, 10)
|
||||
default:
|
||||
glog.V(1).Infof("readStringConfig: unexpected type %T for field %q", value.Kind, field)
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func readBoolConfig(values map[string]*plugin_pb.ConfigValue, field string, fallback bool) bool {
|
||||
if values == nil {
|
||||
return fallback
|
||||
}
|
||||
value := values[field]
|
||||
if value == nil {
|
||||
return fallback
|
||||
}
|
||||
switch kind := value.Kind.(type) {
|
||||
case *plugin_pb.ConfigValue_BoolValue:
|
||||
return kind.BoolValue
|
||||
case *plugin_pb.ConfigValue_StringValue:
|
||||
s := strings.TrimSpace(strings.ToLower(kind.StringValue))
|
||||
if s == "true" || s == "1" || s == "yes" {
|
||||
return true
|
||||
}
|
||||
if s == "false" || s == "0" || s == "no" {
|
||||
return false
|
||||
}
|
||||
glog.V(1).Infof("readBoolConfig: unrecognized string value %q for field %q, using fallback %v", kind.StringValue, field, fallback)
|
||||
case *plugin_pb.ConfigValue_Int64Value:
|
||||
return kind.Int64Value != 0
|
||||
default:
|
||||
glog.V(1).Infof("readBoolConfig: unexpected config value type %T for field %q, using fallback %v", value.Kind, field, fallback)
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func readInt64Config(values map[string]*plugin_pb.ConfigValue, field string, fallback int64) int64 {
|
||||
if values == nil {
|
||||
return fallback
|
||||
}
|
||||
value := values[field]
|
||||
if value == nil {
|
||||
return fallback
|
||||
}
|
||||
switch kind := value.Kind.(type) {
|
||||
case *plugin_pb.ConfigValue_Int64Value:
|
||||
return kind.Int64Value
|
||||
case *plugin_pb.ConfigValue_DoubleValue:
|
||||
return int64(kind.DoubleValue)
|
||||
case *plugin_pb.ConfigValue_StringValue:
|
||||
parsed, err := strconv.ParseInt(strings.TrimSpace(kind.StringValue), 10, 64)
|
||||
if err == nil {
|
||||
return parsed
|
||||
}
|
||||
default:
|
||||
glog.V(1).Infof("readInt64Config: unexpected config value type %T for field %q, using fallback %d", value.Kind, field, fallback)
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
)
|
||||
|
||||
const lifecycleXMLKey = "s3-bucket-lifecycle-configuration-xml"
|
||||
|
||||
// detectBucketsWithLifecycleRules scans all S3 buckets to find those
|
||||
// with lifecycle rules, either TTL entries in filer.conf or lifecycle
|
||||
// XML stored in bucket metadata.
|
||||
func (h *Handler) detectBucketsWithLifecycleRules(
|
||||
ctx context.Context,
|
||||
filerClient filer_pb.SeaweedFilerClient,
|
||||
config Config,
|
||||
bucketFilter string,
|
||||
maxResults int,
|
||||
) ([]*plugin_pb.JobProposal, error) {
|
||||
// Load filer configuration to find TTL rules.
|
||||
fc, err := loadFilerConf(ctx, filerClient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load filer conf: %w", err)
|
||||
}
|
||||
|
||||
bucketsPath := defaultBucketsPath
|
||||
bucketMatchers := wildcard.CompileWildcardMatchers(bucketFilter)
|
||||
|
||||
// List all buckets.
|
||||
bucketEntries, err := listFilerEntries(ctx, filerClient, bucketsPath, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list buckets at %s: %w", bucketsPath, err)
|
||||
}
|
||||
|
||||
var proposals []*plugin_pb.JobProposal
|
||||
for _, entry := range bucketEntries {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return proposals, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
if !entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
bucketName := entry.Name
|
||||
if !wildcard.MatchesAnyWildcard(bucketMatchers, bucketName) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check for lifecycle rules from two sources:
|
||||
// 1. filer.conf TTLs (legacy Expiration.Days fast path)
|
||||
// 2. Stored lifecycle XML in bucket metadata (full rule support)
|
||||
collection := bucketName
|
||||
ttls := fc.GetCollectionTtls(collection)
|
||||
|
||||
hasLifecycleXML := entry.Extended != nil && len(entry.Extended[lifecycleXMLKey]) > 0
|
||||
versioningStatus := ""
|
||||
if entry.Extended != nil {
|
||||
versioningStatus = string(entry.Extended[s3_constants.ExtVersioningKey])
|
||||
}
|
||||
|
||||
ruleCount := int64(len(ttls))
|
||||
if !hasLifecycleXML && ruleCount == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
glog.V(2).Infof("s3_lifecycle: bucket %s has %d TTL rule(s), lifecycle_xml=%v, versioning=%s",
|
||||
bucketName, ruleCount, hasLifecycleXML, versioningStatus)
|
||||
|
||||
proposal := &plugin_pb.JobProposal{
|
||||
ProposalId: fmt.Sprintf("s3_lifecycle:%s", bucketName),
|
||||
JobType: jobType,
|
||||
Summary: fmt.Sprintf("Lifecycle management for bucket %s", bucketName),
|
||||
DedupeKey: fmt.Sprintf("s3_lifecycle:%s", bucketName),
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"bucket": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: bucketName}},
|
||||
"buckets_path": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: bucketsPath}},
|
||||
"collection": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: collection}},
|
||||
"rule_count": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: ruleCount}},
|
||||
"has_lifecycle_xml": {Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: hasLifecycleXML}},
|
||||
"versioning_status": {Kind: &plugin_pb.ConfigValue_StringValue{StringValue: versioningStatus}},
|
||||
},
|
||||
Labels: map[string]string{
|
||||
"bucket": bucketName,
|
||||
},
|
||||
}
|
||||
|
||||
proposals = append(proposals, proposal)
|
||||
if maxResults > 0 && len(proposals) >= maxResults {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return proposals, nil
|
||||
}
|
||||
|
||||
const defaultBucketsPath = "/buckets"
|
||||
|
||||
// loadFilerConf reads the filer configuration from the filer.
|
||||
func loadFilerConf(ctx context.Context, client filer_pb.SeaweedFilerClient) (*filer.FilerConf, error) {
|
||||
fc := filer.NewFilerConf()
|
||||
|
||||
content, err := filer.ReadInsideFiler(ctx, client, filer.DirectoryEtcSeaweedFS, filer.FilerConfName)
|
||||
if err != nil {
|
||||
// filer.conf may not exist yet - return empty config.
|
||||
glog.V(1).Infof("s3_lifecycle: filer.conf not found or unreadable: %v (using empty config)", err)
|
||||
return fc, nil
|
||||
}
|
||||
if err := fc.LoadFromBytes(content); err != nil {
|
||||
return nil, fmt.Errorf("parse filer.conf: %w", err)
|
||||
}
|
||||
|
||||
return fc, nil
|
||||
}
|
||||
|
||||
// listFilerEntries lists directory entries from the filer.
|
||||
func listFilerEntries(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, startFrom string) ([]*filer_pb.Entry, error) {
|
||||
var entries []*filer_pb.Entry
|
||||
err := filer_pb.SeaweedList(ctx, client, dir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
entries = append(entries, entry)
|
||||
return nil
|
||||
}, startFrom, false, 10000)
|
||||
return entries, err
|
||||
}
|
||||
|
||||
type expiredObject struct {
|
||||
dir string
|
||||
name string
|
||||
}
|
||||
|
||||
// listExpiredObjects scans a bucket directory tree for objects whose TTL
|
||||
// has expired based on their TtlSec attribute set by PutBucketLifecycle.
|
||||
func listExpiredObjects(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
bucketsPath, bucket string,
|
||||
limit int64,
|
||||
) ([]expiredObject, int64, error) {
|
||||
var expired []expiredObject
|
||||
var scanned int64
|
||||
|
||||
bucketPath := path.Join(bucketsPath, bucket)
|
||||
|
||||
// Walk the bucket directory tree using breadth-first traversal.
|
||||
dirsToProcess := []string{bucketPath}
|
||||
for len(dirsToProcess) > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return expired, scanned, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
dir := dirsToProcess[0]
|
||||
dirsToProcess = dirsToProcess[1:]
|
||||
|
||||
limitReached := false
|
||||
err := filer_pb.SeaweedList(ctx, client, dir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
if entry.IsDirectory {
|
||||
dirsToProcess = append(dirsToProcess, path.Join(dir, entry.Name))
|
||||
return nil
|
||||
}
|
||||
scanned++
|
||||
|
||||
if isExpiredByTTL(entry) {
|
||||
expired = append(expired, expiredObject{
|
||||
dir: dir,
|
||||
name: entry.Name,
|
||||
})
|
||||
}
|
||||
|
||||
if limit > 0 && int64(len(expired)) >= limit {
|
||||
limitReached = true
|
||||
return fmt.Errorf("limit reached")
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10000)
|
||||
|
||||
if err != nil && !strings.Contains(err.Error(), "limit reached") {
|
||||
return expired, scanned, fmt.Errorf("list %s: %w", dir, err)
|
||||
}
|
||||
|
||||
if limitReached || (limit > 0 && int64(len(expired)) >= limit) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return expired, scanned, nil
|
||||
}
|
||||
|
||||
// isExpiredByTTL checks if an entry is expired based on its TTL attribute.
|
||||
// SeaweedFS sets TtlSec on entries when lifecycle rules are applied via
|
||||
// PutBucketLifecycleConfiguration. An entry is expired when
|
||||
// creation_time + TTL < now.
|
||||
func isExpiredByTTL(entry *filer_pb.Entry) bool {
|
||||
if entry == nil || entry.Attributes == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
ttlSec := entry.Attributes.TtlSec
|
||||
if ttlSec <= 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
crTime := entry.Attributes.Crtime
|
||||
if crTime <= 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
expirationUnix := crTime + int64(ttlSec)
|
||||
return expirationUnix < nowUnix()
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
func TestBucketHasLifecycleXML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extended map[string][]byte
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "has_lifecycle_xml",
|
||||
extended: map[string][]byte{lifecycleXMLKey: []byte("<LifecycleConfiguration/>")},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "empty_lifecycle_xml",
|
||||
extended: map[string][]byte{lifecycleXMLKey: {}},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "no_lifecycle_xml",
|
||||
extended: map[string][]byte{"other-key": []byte("value")},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "nil_extended",
|
||||
extended: nil,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.extended != nil && len(tt.extended[lifecycleXMLKey]) > 0
|
||||
if got != tt.want {
|
||||
t.Errorf("hasLifecycleXML = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketVersioningStatus(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extended map[string][]byte
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "versioning_enabled",
|
||||
extended: map[string][]byte{
|
||||
s3_constants.ExtVersioningKey: []byte("Enabled"),
|
||||
},
|
||||
want: "Enabled",
|
||||
},
|
||||
{
|
||||
name: "versioning_suspended",
|
||||
extended: map[string][]byte{
|
||||
s3_constants.ExtVersioningKey: []byte("Suspended"),
|
||||
},
|
||||
want: "Suspended",
|
||||
},
|
||||
{
|
||||
name: "no_versioning",
|
||||
extended: map[string][]byte{},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "nil_extended",
|
||||
extended: nil,
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var got string
|
||||
if tt.extended != nil {
|
||||
got = string(tt.extended[s3_constants.ExtVersioningKey])
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("versioningStatus = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectionProposalParameters(t *testing.T) {
|
||||
// Verify that bucket entries with lifecycle XML or TTL rules produce
|
||||
// proposals with the expected parameters.
|
||||
t.Run("bucket_with_lifecycle_xml_and_versioning", func(t *testing.T) {
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "my-bucket",
|
||||
IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
lifecycleXMLKey: []byte(`<LifecycleConfiguration><Rule><Status>Enabled</Status></Rule></LifecycleConfiguration>`),
|
||||
s3_constants.ExtVersioningKey: []byte("Enabled"),
|
||||
},
|
||||
}
|
||||
|
||||
hasXML := entry.Extended != nil && len(entry.Extended[lifecycleXMLKey]) > 0
|
||||
versioning := ""
|
||||
if entry.Extended != nil {
|
||||
versioning = string(entry.Extended[s3_constants.ExtVersioningKey])
|
||||
}
|
||||
|
||||
if !hasXML {
|
||||
t.Error("expected hasLifecycleXML=true")
|
||||
}
|
||||
if versioning != "Enabled" {
|
||||
t.Errorf("expected versioning=Enabled, got %q", versioning)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("bucket_without_lifecycle_or_ttl_is_skipped", func(t *testing.T) {
|
||||
entry := &filer_pb.Entry{
|
||||
Name: "empty-bucket",
|
||||
IsDirectory: true,
|
||||
Extended: map[string][]byte{},
|
||||
}
|
||||
|
||||
hasXML := entry.Extended != nil && len(entry.Extended[lifecycleXMLKey]) > 0
|
||||
ttlCount := 0 // simulated: no TTL rules in filer.conf
|
||||
|
||||
if hasXML || ttlCount > 0 {
|
||||
t.Error("expected bucket to be skipped (no lifecycle XML, no TTLs)")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,878 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
var errLimitReached = errors.New("limit reached")
|
||||
|
||||
type executionResult struct {
|
||||
objectsExpired int64
|
||||
objectsScanned int64
|
||||
deleteMarkersClean int64
|
||||
mpuAborted int64
|
||||
errors int64
|
||||
}
|
||||
|
||||
// executeLifecycleForBucket processes lifecycle rules for a single bucket:
|
||||
// 1. Reads filer.conf to get TTL rules for the bucket's collection
|
||||
// 2. Walks the bucket directory tree to find expired objects
|
||||
// 3. Deletes expired objects (unless dry run)
|
||||
func (h *Handler) executeLifecycleForBucket(
|
||||
ctx context.Context,
|
||||
filerClient filer_pb.SeaweedFilerClient,
|
||||
config Config,
|
||||
bucket, bucketsPath string,
|
||||
sender pluginworker.ExecutionSender,
|
||||
jobID string,
|
||||
) (*executionResult, error) {
|
||||
result := &executionResult{}
|
||||
|
||||
// Try to load lifecycle rules from stored XML first (full rule evaluation).
|
||||
// Fall back to filer.conf TTL-only evaluation only if no XML is configured.
|
||||
// If XML exists but is malformed, fail closed (don't fall back to TTL,
|
||||
// which could apply broader rules and delete objects the XML rules would keep).
|
||||
// Transient filer errors fall back to TTL with a warning.
|
||||
lifecycleRules, xmlErr := loadLifecycleRulesFromBucket(ctx, filerClient, bucketsPath, bucket)
|
||||
if xmlErr != nil && errors.Is(xmlErr, errMalformedLifecycleXML) {
|
||||
glog.Errorf("s3_lifecycle: bucket %s: %v (skipping bucket)", bucket, xmlErr)
|
||||
return result, xmlErr
|
||||
}
|
||||
if xmlErr != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: bucket %s: transient error loading lifecycle XML: %v, falling back to TTL", bucket, xmlErr)
|
||||
}
|
||||
// lifecycleRules is non-nil when XML was present (even if empty/all disabled).
|
||||
// Only fall back to TTL when XML was truly absent (nil).
|
||||
xmlPresent := xmlErr == nil && lifecycleRules != nil
|
||||
useRuleEval := xmlPresent && len(lifecycleRules) > 0
|
||||
|
||||
if !useRuleEval && !xmlPresent {
|
||||
// Fall back to filer.conf TTL rules only when no lifecycle XML exists.
|
||||
// When XML is present but has no effective rules, skip TTL fallback.
|
||||
fc, err := loadFilerConf(ctx, filerClient)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("load filer conf: %w", err)
|
||||
}
|
||||
collection := bucket
|
||||
ttlRules := fc.GetCollectionTtls(collection)
|
||||
if len(ttlRules) == 0 {
|
||||
glog.V(1).Infof("s3_lifecycle: bucket %s has no lifecycle rules, skipping", bucket)
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: jobID,
|
||||
JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
ProgressPercent: 10,
|
||||
Stage: "scanning",
|
||||
Message: fmt.Sprintf("scanning bucket %s for expired objects", bucket),
|
||||
})
|
||||
|
||||
// Shared budget across all phases so we don't exceed MaxDeletesPerBucket.
|
||||
remaining := config.MaxDeletesPerBucket
|
||||
|
||||
// Find expired objects using rule-based evaluation or TTL fallback.
|
||||
var expired []expiredObject
|
||||
var scanned int64
|
||||
var err error
|
||||
if useRuleEval {
|
||||
expired, scanned, err = listExpiredObjectsByRules(ctx, filerClient, bucketsPath, bucket, lifecycleRules, remaining)
|
||||
} else if !xmlPresent {
|
||||
// TTL-only scan when no lifecycle XML exists.
|
||||
expired, scanned, err = listExpiredObjects(ctx, filerClient, bucketsPath, bucket, remaining)
|
||||
}
|
||||
// When xmlPresent but no effective rules (all disabled), skip object scanning.
|
||||
result.objectsScanned = scanned
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("list expired objects: %w", err)
|
||||
}
|
||||
|
||||
if len(expired) > 0 {
|
||||
glog.V(1).Infof("s3_lifecycle: bucket %s: found %d expired objects out of %d scanned", bucket, len(expired), scanned)
|
||||
} else {
|
||||
glog.V(1).Infof("s3_lifecycle: bucket %s: scanned %d objects, none expired", bucket, scanned)
|
||||
}
|
||||
|
||||
if config.DryRun && len(expired) > 0 {
|
||||
result.objectsExpired = int64(len(expired))
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: jobID,
|
||||
JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
ProgressPercent: 100,
|
||||
Stage: "dry_run",
|
||||
Message: fmt.Sprintf("dry run: would delete %d expired objects", len(expired)),
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Delete expired objects in batches.
|
||||
if len(expired) > 0 {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: jobID,
|
||||
JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
ProgressPercent: 50,
|
||||
Stage: "deleting",
|
||||
Message: fmt.Sprintf("deleting %d expired objects", len(expired)),
|
||||
})
|
||||
|
||||
var batchSize int
|
||||
if config.BatchSize <= 0 {
|
||||
batchSize = defaultBatchSize
|
||||
} else if config.BatchSize > math.MaxInt {
|
||||
batchSize = math.MaxInt
|
||||
} else {
|
||||
batchSize = int(config.BatchSize)
|
||||
}
|
||||
|
||||
for i := 0; i < len(expired); i += batchSize {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return result, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
end := i + batchSize
|
||||
if end > len(expired) {
|
||||
end = len(expired)
|
||||
}
|
||||
batch := expired[i:end]
|
||||
|
||||
deleted, errs, batchErr := deleteExpiredObjects(ctx, filerClient, batch)
|
||||
result.objectsExpired += int64(deleted)
|
||||
result.errors += int64(errs)
|
||||
|
||||
if batchErr != nil {
|
||||
return result, batchErr
|
||||
}
|
||||
|
||||
progress := float64(end)/float64(len(expired))*50 + 50 // 50-100%
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: jobID,
|
||||
JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
ProgressPercent: progress,
|
||||
Stage: "deleting",
|
||||
Message: fmt.Sprintf("deleted %d/%d expired objects", result.objectsExpired, len(expired)),
|
||||
})
|
||||
}
|
||||
|
||||
// Clean up .versions directories left empty after version deletion.
|
||||
cleanupEmptyVersionsDirectories(ctx, filerClient, expired)
|
||||
|
||||
remaining -= result.objectsExpired + result.errors
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Delete marker cleanup.
|
||||
if config.DeleteMarkerCleanup && remaining > 0 {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: jobID, JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
Stage: "cleaning_delete_markers", Message: "cleaning expired delete markers",
|
||||
})
|
||||
cleaned, cleanErrs, cleanCtxErr := cleanupDeleteMarkers(ctx, filerClient, bucketsPath, bucket, lifecycleRules, remaining)
|
||||
result.deleteMarkersClean = int64(cleaned)
|
||||
result.errors += int64(cleanErrs)
|
||||
if cleanCtxErr != nil {
|
||||
return result, cleanCtxErr
|
||||
}
|
||||
remaining -= int64(cleaned + cleanErrs)
|
||||
if remaining < 0 {
|
||||
remaining = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Abort incomplete multipart uploads.
|
||||
// When lifecycle XML exists, evaluate each upload against the rules
|
||||
// (respecting per-rule prefix filters and DaysAfterInitiation).
|
||||
// Fall back to worker config abort_mpu_days only when no lifecycle
|
||||
// XML is configured for the bucket.
|
||||
if xmlPresent && remaining > 0 {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: jobID, JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
Stage: "aborting_mpus", Message: "evaluating MPU abort rules",
|
||||
})
|
||||
aborted, abortErrs, abortCtxErr := abortMPUsByRules(ctx, filerClient, bucketsPath, bucket, lifecycleRules, remaining)
|
||||
result.mpuAborted = int64(aborted)
|
||||
result.errors += int64(abortErrs)
|
||||
if abortCtxErr != nil {
|
||||
return result, abortCtxErr
|
||||
}
|
||||
} else if !xmlPresent && config.AbortMPUDays > 0 && remaining > 0 {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: jobID, JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING,
|
||||
Stage: "aborting_mpus", Message: fmt.Sprintf("aborting multipart uploads older than %d days", config.AbortMPUDays),
|
||||
})
|
||||
aborted, abortErrs, abortCtxErr := abortIncompleteMPUs(ctx, filerClient, bucketsPath, bucket, config.AbortMPUDays, remaining)
|
||||
result.mpuAborted = int64(aborted)
|
||||
result.errors += int64(abortErrs)
|
||||
if abortCtxErr != nil {
|
||||
return result, abortCtxErr
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// cleanupDeleteMarkers scans versioned objects and removes delete markers
|
||||
// that are the sole remaining version. This matches AWS S3
|
||||
// ExpiredObjectDeleteMarker semantics: a delete marker is only removed when
|
||||
// it is the only version of an object (no non-current versions behind it).
|
||||
//
|
||||
// This phase should run AFTER NoncurrentVersionExpiration (PR 4) so that
|
||||
// non-current versions have already been cleaned up, potentially leaving
|
||||
// delete markers as sole versions eligible for removal.
|
||||
func cleanupDeleteMarkers(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
bucketsPath, bucket string,
|
||||
rules []s3lifecycle.Rule,
|
||||
limit int64,
|
||||
) (cleaned, errors int, ctxErr error) {
|
||||
bucketPath := path.Join(bucketsPath, bucket)
|
||||
|
||||
dirsToProcess := []string{bucketPath}
|
||||
for len(dirsToProcess) > 0 {
|
||||
if ctx.Err() != nil {
|
||||
return cleaned, errors, ctx.Err()
|
||||
}
|
||||
|
||||
dir := dirsToProcess[0]
|
||||
dirsToProcess = dirsToProcess[1:]
|
||||
|
||||
listErr := filer_pb.SeaweedList(ctx, client, dir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
if entry.IsDirectory {
|
||||
if dir == bucketPath && entry.Name == s3_constants.MultipartUploadsFolder {
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) {
|
||||
versionsDir := path.Join(dir, entry.Name)
|
||||
// Check if the latest version is a delete marker.
|
||||
latestIsMarker := string(entry.Extended[s3_constants.ExtLatestVersionIsDeleteMarker]) == "true"
|
||||
if !latestIsMarker {
|
||||
return nil
|
||||
}
|
||||
// Count versions in the directory.
|
||||
versionCount := 0
|
||||
countErr := filer_pb.SeaweedList(ctx, client, versionsDir, "", func(ve *filer_pb.Entry, _ bool) error {
|
||||
if !ve.IsDirectory {
|
||||
versionCount++
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10000)
|
||||
if countErr != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to count versions in %s: %v", versionsDir, countErr)
|
||||
errors++
|
||||
return nil
|
||||
}
|
||||
// Only remove if the delete marker is the sole version.
|
||||
if versionCount != 1 {
|
||||
return nil
|
||||
}
|
||||
// Check that a matching ExpiredObjectDeleteMarker rule exists.
|
||||
// The rule's prefix filter must match this object's key.
|
||||
relDir := strings.TrimPrefix(versionsDir, bucketPath+"/")
|
||||
objKey := strings.TrimSuffix(relDir, s3_constants.VersionsFolder)
|
||||
if len(rules) > 0 && !matchesDeleteMarkerRule(rules, objKey) {
|
||||
return nil
|
||||
}
|
||||
// Find and remove the sole delete marker entry.
|
||||
removedHere := false
|
||||
removeErr := filer_pb.SeaweedList(ctx, client, versionsDir, "", func(ve *filer_pb.Entry, _ bool) error {
|
||||
if !ve.IsDirectory && isDeleteMarker(ve) {
|
||||
if err := filer_pb.DoRemove(ctx, client, versionsDir, ve.Name, true, false, false, false, nil); err != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to remove delete marker %s/%s: %v", versionsDir, ve.Name, err)
|
||||
errors++
|
||||
} else {
|
||||
cleaned++
|
||||
removedHere = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10)
|
||||
if removeErr != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to scan for delete marker in %s: %v", versionsDir, removeErr)
|
||||
}
|
||||
// Remove the now-empty .versions directory only if we
|
||||
// actually deleted the marker in this specific directory.
|
||||
if removedHere {
|
||||
_ = filer_pb.DoRemove(ctx, client, dir, entry.Name, true, true, true, false, nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
dirsToProcess = append(dirsToProcess, path.Join(dir, entry.Name))
|
||||
return nil
|
||||
}
|
||||
|
||||
// For non-versioned objects: only clean up if explicitly a delete marker
|
||||
// and a matching rule exists.
|
||||
relKey := strings.TrimPrefix(path.Join(dir, entry.Name), bucketPath+"/")
|
||||
if isDeleteMarker(entry) && matchesDeleteMarkerRule(rules, relKey) {
|
||||
if err := filer_pb.DoRemove(ctx, client, dir, entry.Name, true, false, false, false, nil); err != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to remove delete marker %s/%s: %v", dir, entry.Name, err)
|
||||
errors++
|
||||
} else {
|
||||
cleaned++
|
||||
}
|
||||
}
|
||||
|
||||
if limit > 0 && int64(cleaned+errors) >= limit {
|
||||
return fmt.Errorf("limit reached")
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10000)
|
||||
|
||||
if listErr != nil && !strings.Contains(listErr.Error(), "limit reached") {
|
||||
return cleaned, errors, fmt.Errorf("list %s: %w", dir, listErr)
|
||||
}
|
||||
|
||||
if limit > 0 && int64(cleaned+errors) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
return cleaned, errors, nil
|
||||
}
|
||||
|
||||
// isDeleteMarker checks if an entry is an S3 delete marker.
|
||||
func isDeleteMarker(entry *filer_pb.Entry) bool {
|
||||
if entry == nil || entry.Extended == nil {
|
||||
return false
|
||||
}
|
||||
return string(entry.Extended[s3_constants.ExtDeleteMarkerKey]) == "true"
|
||||
}
|
||||
|
||||
// matchesDeleteMarkerRule checks if any enabled ExpiredObjectDeleteMarker rule
|
||||
// matches the given object key using the full filter model (prefix, tags, size).
|
||||
// When no lifecycle rules are provided (nil means no XML configured),
|
||||
// falls back to legacy behavior (returns true to allow cleanup).
|
||||
// A non-nil empty slice means XML was present but had no matching rules,
|
||||
// so cleanup is not allowed.
|
||||
func matchesDeleteMarkerRule(rules []s3lifecycle.Rule, objKey string) bool {
|
||||
if rules == nil {
|
||||
return true // legacy fallback: no lifecycle XML configured
|
||||
}
|
||||
// Delete markers have no size or tags, so build a minimal ObjectInfo.
|
||||
obj := s3lifecycle.ObjectInfo{Key: objKey}
|
||||
for _, r := range rules {
|
||||
if r.Status == "Enabled" && r.ExpiredObjectDeleteMarker && s3lifecycle.MatchesFilter(r, obj) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// abortMPUsByRules scans the .uploads directory and evaluates each upload
|
||||
// against lifecycle rules using EvaluateMPUAbort, which respects per-rule
|
||||
// prefix filters and DaysAfterInitiation thresholds.
|
||||
func abortMPUsByRules(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
bucketsPath, bucket string,
|
||||
rules []s3lifecycle.Rule,
|
||||
limit int64,
|
||||
) (aborted, errs int, ctxErr error) {
|
||||
uploadsDir := path.Join(bucketsPath, bucket, ".uploads")
|
||||
now := time.Now()
|
||||
|
||||
listErr := filer_pb.SeaweedList(ctx, client, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if !entry.IsDirectory {
|
||||
return nil
|
||||
}
|
||||
if entry.Attributes == nil || entry.Attributes.Crtime <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
createdAt := time.Unix(entry.Attributes.Crtime, 0)
|
||||
result := s3lifecycle.EvaluateMPUAbort(rules, entry.Name, createdAt, now)
|
||||
if result.Action == s3lifecycle.ActionAbortMultipartUpload {
|
||||
uploadPath := path.Join(uploadsDir, entry.Name)
|
||||
if err := filer_pb.DoRemove(ctx, client, uploadsDir, entry.Name, true, true, true, false, nil); err != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to abort MPU %s: %v", uploadPath, err)
|
||||
errs++
|
||||
} else {
|
||||
aborted++
|
||||
}
|
||||
}
|
||||
|
||||
if limit > 0 && int64(aborted+errs) >= limit {
|
||||
return errLimitReached
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10000)
|
||||
|
||||
if listErr != nil && !errors.Is(listErr, errLimitReached) {
|
||||
return aborted, errs, fmt.Errorf("list uploads in %s: %w", uploadsDir, listErr)
|
||||
}
|
||||
return aborted, errs, nil
|
||||
}
|
||||
|
||||
// abortIncompleteMPUs scans the .uploads directory under a bucket and
|
||||
// removes multipart upload entries older than the specified number of days.
|
||||
func abortIncompleteMPUs(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
bucketsPath, bucket string,
|
||||
olderThanDays, limit int64,
|
||||
) (aborted, errors int, ctxErr error) {
|
||||
uploadsDir := path.Join(bucketsPath, bucket, ".uploads")
|
||||
cutoff := time.Now().Add(-time.Duration(olderThanDays) * 24 * time.Hour)
|
||||
|
||||
listErr := filer_pb.SeaweedList(ctx, client, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if !entry.IsDirectory {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Each subdirectory under .uploads is one multipart upload.
|
||||
// Check the directory creation time.
|
||||
if entry.Attributes != nil && entry.Attributes.Crtime > 0 {
|
||||
created := time.Unix(entry.Attributes.Crtime, 0)
|
||||
if created.Before(cutoff) {
|
||||
uploadPath := path.Join(uploadsDir, entry.Name)
|
||||
if err := filer_pb.DoRemove(ctx, client, uploadsDir, entry.Name, true, true, true, false, nil); err != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to abort MPU %s: %v", uploadPath, err)
|
||||
errors++
|
||||
} else {
|
||||
aborted++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if limit > 0 && int64(aborted+errors) >= limit {
|
||||
return fmt.Errorf("limit reached")
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10000)
|
||||
|
||||
if listErr != nil && !strings.Contains(listErr.Error(), "limit reached") {
|
||||
return aborted, errors, fmt.Errorf("list uploads in %s: %w", uploadsDir, listErr)
|
||||
}
|
||||
|
||||
return aborted, errors, nil
|
||||
}
|
||||
|
||||
// deleteExpiredObjects deletes a batch of expired objects from the filer.
|
||||
// Returns a non-nil error when the context is canceled mid-batch.
|
||||
func deleteExpiredObjects(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
objects []expiredObject,
|
||||
) (deleted, errors int, ctxErr error) {
|
||||
for _, obj := range objects {
|
||||
if ctx.Err() != nil {
|
||||
return deleted, errors, ctx.Err()
|
||||
}
|
||||
|
||||
err := filer_pb.DoRemove(ctx, client, obj.dir, obj.name, true, false, false, false, nil)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to delete %s/%s: %v", obj.dir, obj.name, err)
|
||||
errors++
|
||||
continue
|
||||
}
|
||||
deleted++
|
||||
}
|
||||
return deleted, errors, nil
|
||||
}
|
||||
|
||||
// nowUnix returns the current time as a Unix timestamp.
|
||||
func nowUnix() int64 {
|
||||
return time.Now().Unix()
|
||||
}
|
||||
|
||||
// listExpiredObjectsByRules scans a bucket directory tree and evaluates
|
||||
// lifecycle rules against each object using the s3lifecycle evaluator.
|
||||
// This function handles non-versioned objects (IsLatest=true). Versioned
|
||||
// objects in .versions directories are handled by processVersionsDirectory
|
||||
// (added in a separate change for NoncurrentVersionExpiration support).
|
||||
func listExpiredObjectsByRules(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
bucketsPath, bucket string,
|
||||
rules []s3lifecycle.Rule,
|
||||
limit int64,
|
||||
) ([]expiredObject, int64, error) {
|
||||
var expired []expiredObject
|
||||
var scanned int64
|
||||
|
||||
bucketPath := path.Join(bucketsPath, bucket)
|
||||
now := time.Now()
|
||||
needTags := s3lifecycle.HasTagRules(rules)
|
||||
|
||||
dirsToProcess := []string{bucketPath}
|
||||
for len(dirsToProcess) > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return expired, scanned, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
dir := dirsToProcess[0]
|
||||
dirsToProcess = dirsToProcess[1:]
|
||||
|
||||
limitReached := false
|
||||
err := filer_pb.SeaweedList(ctx, client, dir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
if entry.IsDirectory {
|
||||
if dir == bucketPath && entry.Name == s3_constants.MultipartUploadsFolder {
|
||||
return nil // skip .uploads at bucket root only
|
||||
}
|
||||
if strings.HasSuffix(entry.Name, s3_constants.VersionsFolder) {
|
||||
versionsDir := path.Join(dir, entry.Name)
|
||||
|
||||
// Evaluate Expiration rules against the latest version.
|
||||
// In versioned buckets, data lives in .versions/ directories,
|
||||
// so we must evaluate the latest version here — it is never
|
||||
// seen as a regular file entry in the parent directory.
|
||||
if obj, ok := latestVersionExpiredByRules(ctx, client, entry, versionsDir, bucketPath, rules, now, needTags); ok {
|
||||
expired = append(expired, obj)
|
||||
scanned++
|
||||
if limit > 0 && int64(len(expired)) >= limit {
|
||||
limitReached = true
|
||||
return errLimitReached
|
||||
}
|
||||
}
|
||||
|
||||
// Process noncurrent versions.
|
||||
vExpired, vScanned, vErr := processVersionsDirectory(ctx, client, versionsDir, bucketPath, rules, now, needTags, limit-int64(len(expired)))
|
||||
if vErr != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: %v", vErr)
|
||||
return vErr
|
||||
}
|
||||
expired = append(expired, vExpired...)
|
||||
scanned += vScanned
|
||||
if limit > 0 && int64(len(expired)) >= limit {
|
||||
limitReached = true
|
||||
return errLimitReached
|
||||
}
|
||||
return nil
|
||||
}
|
||||
dirsToProcess = append(dirsToProcess, path.Join(dir, entry.Name))
|
||||
return nil
|
||||
}
|
||||
scanned++
|
||||
|
||||
// Skip objects already handled by TTL fast path.
|
||||
if entry.Attributes != nil && entry.Attributes.TtlSec > 0 {
|
||||
expirationUnix := entry.Attributes.Crtime + int64(entry.Attributes.TtlSec)
|
||||
if expirationUnix > nowUnix() {
|
||||
return nil // will be expired by RocksDB compaction
|
||||
}
|
||||
}
|
||||
|
||||
// Build ObjectInfo for the evaluator.
|
||||
relKey := strings.TrimPrefix(path.Join(dir, entry.Name), bucketPath+"/")
|
||||
objInfo := s3lifecycle.ObjectInfo{
|
||||
Key: relKey,
|
||||
IsLatest: true, // non-versioned objects are always "latest"
|
||||
}
|
||||
if entry.Attributes != nil {
|
||||
objInfo.Size = int64(entry.Attributes.GetFileSize())
|
||||
if entry.Attributes.Mtime > 0 {
|
||||
objInfo.ModTime = time.Unix(entry.Attributes.Mtime, 0)
|
||||
} else if entry.Attributes.Crtime > 0 {
|
||||
objInfo.ModTime = time.Unix(entry.Attributes.Crtime, 0)
|
||||
}
|
||||
}
|
||||
if needTags {
|
||||
objInfo.Tags = s3lifecycle.ExtractTags(entry.Extended)
|
||||
}
|
||||
|
||||
result := s3lifecycle.Evaluate(rules, objInfo, now)
|
||||
if result.Action == s3lifecycle.ActionDeleteObject {
|
||||
expired = append(expired, expiredObject{dir: dir, name: entry.Name})
|
||||
}
|
||||
|
||||
if limit > 0 && int64(len(expired)) >= limit {
|
||||
limitReached = true
|
||||
return errLimitReached
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10000)
|
||||
|
||||
if err != nil && !errors.Is(err, errLimitReached) {
|
||||
return expired, scanned, fmt.Errorf("list %s: %w", dir, err)
|
||||
}
|
||||
|
||||
if limitReached || (limit > 0 && int64(len(expired)) >= limit) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return expired, scanned, nil
|
||||
}
|
||||
|
||||
// processVersionsDirectory evaluates NoncurrentVersionExpiration rules
|
||||
// against all versions in a .versions directory.
|
||||
func processVersionsDirectory(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
versionsDir, bucketPath string,
|
||||
rules []s3lifecycle.Rule,
|
||||
now time.Time,
|
||||
needTags bool,
|
||||
limit int64,
|
||||
) ([]expiredObject, int64, error) {
|
||||
var expired []expiredObject
|
||||
var scanned int64
|
||||
|
||||
// Check if any rule has NoncurrentVersionExpiration.
|
||||
hasNoncurrentRules := false
|
||||
for _, r := range rules {
|
||||
if r.Status == "Enabled" && r.NoncurrentVersionExpirationDays > 0 {
|
||||
hasNoncurrentRules = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasNoncurrentRules {
|
||||
return nil, 0, nil
|
||||
}
|
||||
|
||||
// List all versions in this directory.
|
||||
var versions []*filer_pb.Entry
|
||||
listErr := filer_pb.SeaweedList(ctx, client, versionsDir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
if !entry.IsDirectory {
|
||||
versions = append(versions, entry)
|
||||
}
|
||||
return nil
|
||||
}, "", false, 10000)
|
||||
if listErr != nil {
|
||||
return nil, 0, fmt.Errorf("list versions in %s: %w", versionsDir, listErr)
|
||||
}
|
||||
if len(versions) <= 1 {
|
||||
return nil, 0, nil // only one version (the latest), nothing to expire
|
||||
}
|
||||
|
||||
// Sort by version timestamp, newest first.
|
||||
sortVersionsByVersionId(versions)
|
||||
|
||||
// Derive the object key from the .versions directory path.
|
||||
// e.g., /buckets/mybucket/path/to/key.versions -> path/to/key
|
||||
relDir := strings.TrimPrefix(versionsDir, bucketPath+"/")
|
||||
objKey := strings.TrimSuffix(relDir, s3_constants.VersionsFolder)
|
||||
|
||||
// Walk versions: first is latest, rest are non-current.
|
||||
noncurrentIndex := 0
|
||||
for i := 1; i < len(versions); i++ {
|
||||
entry := versions[i]
|
||||
scanned++
|
||||
|
||||
// Skip delete markers from expiration evaluation, but count
|
||||
// them toward NewerNoncurrentVersions so data versions get
|
||||
// the correct noncurrent index.
|
||||
if isDeleteMarker(entry) {
|
||||
noncurrentIndex++
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine successor's timestamp (the version that replaced this one).
|
||||
successorEntry := versions[i-1]
|
||||
successorVersionId := strings.TrimPrefix(successorEntry.Name, "v_")
|
||||
successorTime := s3lifecycle.GetVersionTimestamp(successorVersionId)
|
||||
if successorTime.IsZero() && successorEntry.Attributes != nil && successorEntry.Attributes.Mtime > 0 {
|
||||
successorTime = time.Unix(successorEntry.Attributes.Mtime, 0)
|
||||
}
|
||||
|
||||
objInfo := s3lifecycle.ObjectInfo{
|
||||
Key: objKey,
|
||||
IsLatest: false,
|
||||
SuccessorModTime: successorTime,
|
||||
NumVersions: len(versions),
|
||||
NoncurrentIndex: noncurrentIndex,
|
||||
}
|
||||
if entry.Attributes != nil {
|
||||
objInfo.Size = int64(entry.Attributes.GetFileSize())
|
||||
if entry.Attributes.Mtime > 0 {
|
||||
objInfo.ModTime = time.Unix(entry.Attributes.Mtime, 0)
|
||||
}
|
||||
}
|
||||
if needTags {
|
||||
objInfo.Tags = s3lifecycle.ExtractTags(entry.Extended)
|
||||
}
|
||||
|
||||
// Evaluate using the detailed ShouldExpireNoncurrentVersion which
|
||||
// handles NewerNoncurrentVersions.
|
||||
for _, rule := range rules {
|
||||
if s3lifecycle.ShouldExpireNoncurrentVersion(rule, objInfo, noncurrentIndex, now) {
|
||||
expired = append(expired, expiredObject{dir: versionsDir, name: entry.Name})
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
noncurrentIndex++
|
||||
|
||||
if limit > 0 && int64(len(expired)) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return expired, scanned, nil
|
||||
}
|
||||
|
||||
// latestVersionExpiredByRules evaluates Expiration rules (Days/Date) against
|
||||
// the latest version in a .versions directory. In versioned buckets all data
|
||||
// lives inside .versions/ directories, so the latest version is never seen as
|
||||
// a regular file entry during the bucket walk. Without this check, Expiration
|
||||
// rules would never fire for versioned objects (issue #8757).
|
||||
//
|
||||
// The .versions directory entry caches metadata about the latest version in
|
||||
// its Extended attributes, so we can evaluate expiration without an extra
|
||||
// filer round-trip.
|
||||
func latestVersionExpiredByRules(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
dirEntry *filer_pb.Entry,
|
||||
versionsDir, bucketPath string,
|
||||
rules []s3lifecycle.Rule,
|
||||
now time.Time,
|
||||
needTags bool,
|
||||
) (expiredObject, bool) {
|
||||
if dirEntry.Extended == nil {
|
||||
return expiredObject{}, false
|
||||
}
|
||||
|
||||
// Skip if the latest version is a delete marker — those are handled
|
||||
// by the ExpiredObjectDeleteMarker rule in cleanupDeleteMarkers.
|
||||
if string(dirEntry.Extended[s3_constants.ExtLatestVersionIsDeleteMarker]) == "true" {
|
||||
return expiredObject{}, false
|
||||
}
|
||||
|
||||
latestFileName := string(dirEntry.Extended[s3_constants.ExtLatestVersionFileNameKey])
|
||||
if latestFileName == "" {
|
||||
return expiredObject{}, false
|
||||
}
|
||||
|
||||
// Derive the object key: /buckets/b/path/key.versions → path/key
|
||||
relDir := strings.TrimPrefix(versionsDir, bucketPath+"/")
|
||||
objKey := strings.TrimSuffix(relDir, s3_constants.VersionsFolder)
|
||||
|
||||
objInfo := s3lifecycle.ObjectInfo{
|
||||
Key: objKey,
|
||||
IsLatest: true,
|
||||
}
|
||||
|
||||
// Populate ModTime from cached metadata.
|
||||
if mtimeStr := string(dirEntry.Extended[s3_constants.ExtLatestVersionMtimeKey]); mtimeStr != "" {
|
||||
if mtime, err := strconv.ParseInt(mtimeStr, 10, 64); err == nil {
|
||||
objInfo.ModTime = time.Unix(mtime, 0)
|
||||
}
|
||||
}
|
||||
if objInfo.ModTime.IsZero() && dirEntry.Attributes != nil && dirEntry.Attributes.Mtime > 0 {
|
||||
objInfo.ModTime = time.Unix(dirEntry.Attributes.Mtime, 0)
|
||||
}
|
||||
|
||||
// Populate Size from cached metadata.
|
||||
if sizeStr := string(dirEntry.Extended[s3_constants.ExtLatestVersionSizeKey]); sizeStr != "" {
|
||||
if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil {
|
||||
objInfo.Size = size
|
||||
}
|
||||
}
|
||||
|
||||
if needTags {
|
||||
// Tags are stored on the version file entry, not the .versions
|
||||
// directory. Fetch the actual version file to get them.
|
||||
resp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: versionsDir,
|
||||
Name: latestFileName,
|
||||
})
|
||||
if err == nil && resp.Entry != nil {
|
||||
objInfo.Tags = s3lifecycle.ExtractTags(resp.Entry.Extended)
|
||||
}
|
||||
}
|
||||
|
||||
result := s3lifecycle.Evaluate(rules, objInfo, now)
|
||||
if result.Action == s3lifecycle.ActionDeleteObject {
|
||||
return expiredObject{dir: versionsDir, name: latestFileName}, true
|
||||
}
|
||||
|
||||
return expiredObject{}, false
|
||||
}
|
||||
|
||||
// cleanupEmptyVersionsDirectories removes .versions directories that became
|
||||
// empty after their contents were deleted. This is called after
|
||||
// deleteExpiredObjects to avoid leaving orphaned directories.
|
||||
func cleanupEmptyVersionsDirectories(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
deleted []expiredObject,
|
||||
) int {
|
||||
// Collect unique .versions directories that had entries deleted.
|
||||
versionsDirs := map[string]struct{}{}
|
||||
for _, obj := range deleted {
|
||||
if strings.HasSuffix(obj.dir, s3_constants.VersionsFolder) {
|
||||
versionsDirs[obj.dir] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
cleaned := 0
|
||||
for vDir := range versionsDirs {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
// Check if the directory is now empty.
|
||||
empty := true
|
||||
listErr := filer_pb.SeaweedList(ctx, client, vDir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
empty = false
|
||||
return errLimitReached // stop after first entry
|
||||
}, "", false, 1)
|
||||
|
||||
if listErr != nil && !errors.Is(listErr, errLimitReached) {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to check if versions dir %s is empty: %v", vDir, listErr)
|
||||
continue
|
||||
}
|
||||
|
||||
if !empty {
|
||||
continue
|
||||
}
|
||||
|
||||
// Remove the empty .versions directory.
|
||||
parentDir, dirName := path.Split(vDir)
|
||||
parentDir = strings.TrimSuffix(parentDir, "/")
|
||||
if err := filer_pb.DoRemove(ctx, client, parentDir, dirName, false, true, true, false, nil); err != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: failed to clean up empty versions dir %s: %v", vDir, err)
|
||||
} else {
|
||||
cleaned++
|
||||
}
|
||||
}
|
||||
return cleaned
|
||||
}
|
||||
|
||||
// sortVersionsByVersionId sorts version entries newest-first using full
|
||||
// version ID comparison (matching compareVersionIds in s3api_version_id.go).
|
||||
// This uses the complete version ID string, not just the decoded timestamp,
|
||||
// so entries with the same timestamp prefix are correctly ordered by their
|
||||
// random suffix.
|
||||
func sortVersionsByVersionId(versions []*filer_pb.Entry) {
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
vidI := strings.TrimPrefix(versions[i].Name, "v_")
|
||||
vidJ := strings.TrimPrefix(versions[j].Name, "v_")
|
||||
return s3lifecycle.CompareVersionIds(vidI, vidJ) < 0
|
||||
})
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
func TestMatchesDeleteMarkerRule(t *testing.T) {
|
||||
t.Run("nil_rules_legacy_fallback", func(t *testing.T) {
|
||||
if !matchesDeleteMarkerRule(nil, "any/key") {
|
||||
t.Error("nil rules should return true (legacy fallback)")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty_rules_xml_present_no_match", func(t *testing.T) {
|
||||
rules := []s3lifecycle.Rule{}
|
||||
if matchesDeleteMarkerRule(rules, "any/key") {
|
||||
t.Error("empty rules (XML present) should return false")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("matching_prefix_rule", func(t *testing.T) {
|
||||
rules := []s3lifecycle.Rule{
|
||||
{ID: "cleanup", Status: "Enabled", Prefix: "logs/", ExpiredObjectDeleteMarker: true},
|
||||
}
|
||||
if !matchesDeleteMarkerRule(rules, "logs/app.log") {
|
||||
t.Error("should match rule with matching prefix")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non_matching_prefix", func(t *testing.T) {
|
||||
rules := []s3lifecycle.Rule{
|
||||
{ID: "cleanup", Status: "Enabled", Prefix: "logs/", ExpiredObjectDeleteMarker: true},
|
||||
}
|
||||
if matchesDeleteMarkerRule(rules, "data/file.txt") {
|
||||
t.Error("should not match rule with non-matching prefix")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("disabled_rule", func(t *testing.T) {
|
||||
rules := []s3lifecycle.Rule{
|
||||
{ID: "cleanup", Status: "Disabled", ExpiredObjectDeleteMarker: true},
|
||||
}
|
||||
if matchesDeleteMarkerRule(rules, "any/key") {
|
||||
t.Error("disabled rule should not match")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rule_without_delete_marker_flag", func(t *testing.T) {
|
||||
rules := []s3lifecycle.Rule{
|
||||
{ID: "expire", Status: "Enabled", ExpirationDays: 30},
|
||||
}
|
||||
if matchesDeleteMarkerRule(rules, "any/key") {
|
||||
t.Error("rule without ExpiredObjectDeleteMarker should not match")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("tag_filtered_rule_no_tags_on_marker", func(t *testing.T) {
|
||||
rules := []s3lifecycle.Rule{
|
||||
{
|
||||
ID: "tagged", Status: "Enabled",
|
||||
ExpiredObjectDeleteMarker: true,
|
||||
FilterTags: map[string]string{"env": "dev"},
|
||||
},
|
||||
}
|
||||
// Delete markers have no tags, so a tag-filtered rule should not match.
|
||||
if matchesDeleteMarkerRule(rules, "any/key") {
|
||||
t.Error("tag-filtered rule should not match delete marker (no tags)")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,380 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func init() {
|
||||
pluginworker.RegisterHandler(pluginworker.HandlerFactory{
|
||||
JobType: jobType,
|
||||
Category: pluginworker.CategoryHeavy,
|
||||
Aliases: []string{"lifecycle", "s3-lifecycle", "s3.lifecycle"},
|
||||
Build: func(opts pluginworker.HandlerBuildOptions) (pluginworker.JobHandler, error) {
|
||||
return NewHandler(opts.GrpcDialOption), nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Handler implements the JobHandler interface for S3 lifecycle management:
|
||||
// object expiration, delete marker cleanup, and abort incomplete multipart uploads.
|
||||
type Handler struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
}
|
||||
|
||||
const filerConnectTimeout = 5 * time.Second
|
||||
|
||||
// NewHandler creates a new handler for S3 lifecycle management.
|
||||
func NewHandler(grpcDialOption grpc.DialOption) *Handler {
|
||||
return &Handler{grpcDialOption: grpcDialOption}
|
||||
}
|
||||
|
||||
func (h *Handler) Capability() *plugin_pb.JobTypeCapability {
|
||||
return &plugin_pb.JobTypeCapability{
|
||||
JobType: jobType,
|
||||
CanDetect: true,
|
||||
CanExecute: true,
|
||||
MaxDetectionConcurrency: 1,
|
||||
MaxExecutionConcurrency: 4,
|
||||
DisplayName: "S3 Lifecycle",
|
||||
Description: "Manages S3 object lifecycle: expiration of objects based on TTL rules, delete marker cleanup, and abort of incomplete multipart uploads",
|
||||
Weight: 40,
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
return &plugin_pb.JobTypeDescriptor{
|
||||
JobType: jobType,
|
||||
DisplayName: "S3 Lifecycle Management",
|
||||
Description: "Automated S3 object lifecycle management: expire objects by TTL rules, clean up expired delete markers, and abort stale multipart uploads",
|
||||
Icon: "fas fa-hourglass-half",
|
||||
DescriptorVersion: 1,
|
||||
AdminConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "s3-lifecycle-admin",
|
||||
Title: "S3 Lifecycle Admin Config",
|
||||
Description: "Admin-side controls for S3 lifecycle management scope.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "scope",
|
||||
Title: "Scope",
|
||||
Description: "Which buckets to include in lifecycle management.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "bucket_filter",
|
||||
Label: "Bucket Filter",
|
||||
Description: "Wildcard pattern for bucket names to include (e.g. \"prod-*\"). Empty means all buckets.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
WorkerConfigForm: &plugin_pb.ConfigForm{
|
||||
FormId: "s3-lifecycle-worker",
|
||||
Title: "S3 Lifecycle Worker Config",
|
||||
Description: "Worker-side controls for lifecycle execution behavior.",
|
||||
Sections: []*plugin_pb.ConfigSection{
|
||||
{
|
||||
SectionId: "execution",
|
||||
Title: "Execution",
|
||||
Description: "Controls for lifecycle rule execution.",
|
||||
Fields: []*plugin_pb.ConfigField{
|
||||
{
|
||||
Name: "batch_size",
|
||||
Label: "Batch Size",
|
||||
Description: "Number of entries to process per filer listing page.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: configInt64(100),
|
||||
MaxValue: configInt64(10000),
|
||||
},
|
||||
{
|
||||
Name: "max_deletes_per_bucket",
|
||||
Label: "Max Deletes Per Bucket",
|
||||
Description: "Maximum number of expired objects to delete per bucket in one execution run.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: configInt64(100),
|
||||
MaxValue: configInt64(1000000),
|
||||
},
|
||||
{
|
||||
Name: "dry_run",
|
||||
Label: "Dry Run",
|
||||
Description: "When enabled, detect expired objects but do not delete them.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_BOOL,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TOGGLE,
|
||||
},
|
||||
{
|
||||
Name: "delete_marker_cleanup",
|
||||
Label: "Delete Marker Cleanup",
|
||||
Description: "Remove expired delete markers that have no non-current versions.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_BOOL,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TOGGLE,
|
||||
},
|
||||
{
|
||||
Name: "abort_mpu_days",
|
||||
Label: "Abort Incomplete MPU (days)",
|
||||
Description: "Abort incomplete multipart uploads older than this many days. 0 disables.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: configInt64(0),
|
||||
MaxValue: configInt64(365),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
Enabled: true,
|
||||
DetectionIntervalSeconds: 300, // 5 minutes
|
||||
DetectionTimeoutSeconds: 60,
|
||||
MaxJobsPerDetection: 100,
|
||||
GlobalExecutionConcurrency: 2,
|
||||
PerWorkerExecutionConcurrency: 2,
|
||||
RetryLimit: 1,
|
||||
RetryBackoffSeconds: 10,
|
||||
},
|
||||
WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"batch_size": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultBatchSize}},
|
||||
"max_deletes_per_bucket": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultMaxDeletesPerBucket}},
|
||||
"dry_run": {Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: defaultDryRun}},
|
||||
"delete_marker_cleanup": {Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: defaultDeleteMarkerCleanup}},
|
||||
"abort_mpu_days": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultAbortMPUDaysDefault}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) Detect(ctx context.Context, req *plugin_pb.RunDetectionRequest, sender pluginworker.DetectionSender) error {
|
||||
if req == nil {
|
||||
return fmt.Errorf("nil detection request")
|
||||
}
|
||||
|
||||
config := ParseConfig(req.WorkerConfigValues)
|
||||
|
||||
bucketFilter := readStringConfig(req.AdminConfigValues, "bucket_filter", "")
|
||||
|
||||
filerAddresses := filerAddressesFromCluster(req.ClusterContext)
|
||||
if len(filerAddresses) == 0 {
|
||||
_ = sender.SendActivity(pluginworker.BuildDetectorActivity("skipped", "no filer addresses in cluster context", nil))
|
||||
return sendEmptyDetection(sender)
|
||||
}
|
||||
|
||||
_ = sender.SendActivity(pluginworker.BuildDetectorActivity("connecting", "connecting to filer", nil))
|
||||
|
||||
filerClient, filerConn, err := connectToFiler(ctx, filerAddresses, h.grpcDialOption)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to any filer: %v", err)
|
||||
}
|
||||
defer filerConn.Close()
|
||||
|
||||
maxResults := int(req.MaxResults)
|
||||
if maxResults <= 0 {
|
||||
maxResults = 100
|
||||
}
|
||||
|
||||
_ = sender.SendActivity(pluginworker.BuildDetectorActivity("scanning", "scanning buckets for lifecycle rules", nil))
|
||||
proposals, err := h.detectBucketsWithLifecycleRules(ctx, filerClient, config, bucketFilter, maxResults)
|
||||
if err != nil {
|
||||
_ = sender.SendActivity(pluginworker.BuildDetectorActivity("scan_error", fmt.Sprintf("error scanning buckets: %v", err), nil))
|
||||
return fmt.Errorf("detect lifecycle rules: %w", err)
|
||||
}
|
||||
|
||||
_ = sender.SendActivity(pluginworker.BuildDetectorActivity("scan_complete",
|
||||
fmt.Sprintf("found %d bucket(s) with lifecycle rules", len(proposals)),
|
||||
map[string]*plugin_pb.ConfigValue{
|
||||
"buckets_found": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(proposals))}},
|
||||
}))
|
||||
|
||||
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
|
||||
JobType: jobType,
|
||||
Proposals: proposals,
|
||||
HasMore: len(proposals) >= maxResults,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sender.SendComplete(&plugin_pb.DetectionComplete{
|
||||
JobType: jobType,
|
||||
Success: true,
|
||||
TotalProposals: int32(len(proposals)),
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Handler) Execute(ctx context.Context, req *plugin_pb.ExecuteJobRequest, sender pluginworker.ExecutionSender) error {
|
||||
if req == nil || req.Job == nil {
|
||||
return fmt.Errorf("nil execution request")
|
||||
}
|
||||
|
||||
job := req.Job
|
||||
config := ParseConfig(req.WorkerConfigValues)
|
||||
|
||||
bucket := readParamString(job.Parameters, "bucket")
|
||||
bucketsPath := readParamString(job.Parameters, "buckets_path")
|
||||
if bucket == "" || bucketsPath == "" {
|
||||
return fmt.Errorf("missing bucket or buckets_path parameter")
|
||||
}
|
||||
|
||||
filerAddresses := filerAddressesFromCluster(req.ClusterContext)
|
||||
if len(filerAddresses) == 0 {
|
||||
return fmt.Errorf("no filer addresses in cluster context")
|
||||
}
|
||||
|
||||
filerClient, filerConn, err := connectToFiler(ctx, filerAddresses, h.grpcDialOption)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to any filer: %v", err)
|
||||
}
|
||||
defer filerConn.Close()
|
||||
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: job.JobId,
|
||||
JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_ASSIGNED,
|
||||
ProgressPercent: 0,
|
||||
Stage: "starting",
|
||||
Message: fmt.Sprintf("executing lifecycle rules for bucket %s", bucket),
|
||||
})
|
||||
|
||||
start := time.Now()
|
||||
result, execErr := h.executeLifecycleForBucket(ctx, filerClient, config, bucket, bucketsPath, sender, job.JobId)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
metrics := map[string]*plugin_pb.ConfigValue{
|
||||
MetricDurationMs: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: elapsed.Milliseconds()}},
|
||||
}
|
||||
if result != nil {
|
||||
metrics[MetricObjectsExpired] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: result.objectsExpired}}
|
||||
metrics[MetricObjectsScanned] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: result.objectsScanned}}
|
||||
metrics[MetricDeleteMarkersClean] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: result.deleteMarkersClean}}
|
||||
metrics[MetricMPUAborted] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: result.mpuAborted}}
|
||||
metrics[MetricErrors] = &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: result.errors}}
|
||||
}
|
||||
|
||||
var scanned, expired int64
|
||||
if result != nil {
|
||||
scanned = result.objectsScanned
|
||||
expired = result.objectsExpired
|
||||
}
|
||||
|
||||
success := execErr == nil && (result == nil || result.errors == 0)
|
||||
message := fmt.Sprintf("bucket %s: scanned %d objects, expired %d", bucket, scanned, expired)
|
||||
if result != nil && result.deleteMarkersClean > 0 {
|
||||
message += fmt.Sprintf(", delete markers cleaned %d", result.deleteMarkersClean)
|
||||
}
|
||||
if result != nil && result.mpuAborted > 0 {
|
||||
message += fmt.Sprintf(", MPUs aborted %d", result.mpuAborted)
|
||||
}
|
||||
if config.DryRun {
|
||||
message += " (dry run)"
|
||||
}
|
||||
if result != nil && result.errors > 0 {
|
||||
message += fmt.Sprintf(" (%d errors)", result.errors)
|
||||
}
|
||||
if execErr != nil {
|
||||
message = fmt.Sprintf("lifecycle execution failed for bucket %s: %v", bucket, execErr)
|
||||
}
|
||||
|
||||
errMsg := ""
|
||||
if execErr != nil {
|
||||
errMsg = execErr.Error()
|
||||
} else if result != nil && result.errors > 0 {
|
||||
errMsg = fmt.Sprintf("%d objects failed to process", result.errors)
|
||||
}
|
||||
|
||||
return sender.SendCompleted(&plugin_pb.JobCompleted{
|
||||
JobId: job.JobId,
|
||||
JobType: jobType,
|
||||
Success: success,
|
||||
ErrorMessage: errMsg,
|
||||
Result: &plugin_pb.JobResult{
|
||||
Summary: message,
|
||||
OutputValues: metrics,
|
||||
},
|
||||
CompletedAt: timestamppb.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
func connectToFiler(ctx context.Context, addresses []string, dialOption grpc.DialOption) (filer_pb.SeaweedFilerClient, *grpc.ClientConn, error) {
|
||||
var lastErr error
|
||||
for _, addr := range addresses {
|
||||
grpcAddr := pb.ServerAddress(addr).ToGrpcAddress()
|
||||
connCtx, cancel := context.WithTimeout(ctx, filerConnectTimeout)
|
||||
conn, err := pb.GrpcDial(connCtx, grpcAddr, false, dialOption)
|
||||
cancel()
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
glog.V(1).Infof("s3_lifecycle: failed to connect to filer %s (grpc %s): %v", addr, grpcAddr, err)
|
||||
continue
|
||||
}
|
||||
// Verify the connection with a ping.
|
||||
client := filer_pb.NewSeaweedFilerClient(conn)
|
||||
pingCtx, pingCancel := context.WithTimeout(ctx, filerConnectTimeout)
|
||||
_, pingErr := client.Ping(pingCtx, &filer_pb.PingRequest{})
|
||||
pingCancel()
|
||||
if pingErr != nil {
|
||||
_ = conn.Close()
|
||||
lastErr = pingErr
|
||||
glog.V(1).Infof("s3_lifecycle: filer %s ping failed: %v", grpcAddr, pingErr)
|
||||
continue
|
||||
}
|
||||
return client, conn, nil
|
||||
}
|
||||
return nil, nil, lastErr
|
||||
}
|
||||
|
||||
func sendEmptyDetection(sender pluginworker.DetectionSender) error {
|
||||
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
|
||||
JobType: jobType,
|
||||
Proposals: []*plugin_pb.JobProposal{},
|
||||
HasMore: false,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return sender.SendComplete(&plugin_pb.DetectionComplete{
|
||||
JobType: jobType,
|
||||
Success: true,
|
||||
TotalProposals: 0,
|
||||
})
|
||||
}
|
||||
|
||||
func filerAddressesFromCluster(cc *plugin_pb.ClusterContext) []string {
|
||||
if cc == nil {
|
||||
return nil
|
||||
}
|
||||
var addrs []string
|
||||
for _, addr := range cc.FilerGrpcAddresses {
|
||||
trimmed := strings.TrimSpace(addr)
|
||||
if trimmed != "" {
|
||||
addrs = append(addrs, trimmed)
|
||||
}
|
||||
}
|
||||
return addrs
|
||||
}
|
||||
|
||||
func readParamString(params map[string]*plugin_pb.ConfigValue, key string) string {
|
||||
if params == nil {
|
||||
return ""
|
||||
}
|
||||
v := params[key]
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
if sv, ok := v.Kind.(*plugin_pb.ConfigValue_StringValue); ok {
|
||||
return sv.StringValue
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func configInt64(v int64) *plugin_pb.ConfigValue {
|
||||
return &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: v}}
|
||||
}
|
||||
@@ -1,781 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// testFilerServer is an in-memory filer gRPC server for integration tests.
|
||||
type testFilerServer struct {
|
||||
filer_pb.UnimplementedSeaweedFilerServer
|
||||
mu sync.RWMutex
|
||||
entries map[string]*filer_pb.Entry // key: "dir\x00name"
|
||||
}
|
||||
|
||||
func newTestFilerServer() *testFilerServer {
|
||||
return &testFilerServer{entries: make(map[string]*filer_pb.Entry)}
|
||||
}
|
||||
|
||||
func (s *testFilerServer) key(dir, name string) string { return dir + "\x00" + name }
|
||||
|
||||
func (s *testFilerServer) splitKey(key string) (string, string) {
|
||||
for i := range key {
|
||||
if key[i] == '\x00' {
|
||||
return key[:i], key[i+1:]
|
||||
}
|
||||
}
|
||||
return key, ""
|
||||
}
|
||||
|
||||
func (s *testFilerServer) putEntry(dir string, entry *filer_pb.Entry) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.entries[s.key(dir, entry.Name)] = proto.Clone(entry).(*filer_pb.Entry)
|
||||
}
|
||||
|
||||
func (s *testFilerServer) getEntry(dir, name string) *filer_pb.Entry {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
e := s.entries[s.key(dir, name)]
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
return proto.Clone(e).(*filer_pb.Entry)
|
||||
}
|
||||
|
||||
func (s *testFilerServer) hasEntry(dir, name string) bool {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
_, ok := s.entries[s.key(dir, name)]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (s *testFilerServer) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
entry, found := s.entries[s.key(req.Directory, req.Name)]
|
||||
if !found {
|
||||
return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
|
||||
}
|
||||
return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil
|
||||
}
|
||||
|
||||
func (s *testFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
|
||||
// Snapshot entries under lock, then stream without holding the lock
|
||||
// (streaming callbacks may trigger DeleteEntry which needs a write lock).
|
||||
s.mu.RLock()
|
||||
var names []string
|
||||
for key := range s.entries {
|
||||
dir, name := s.splitKey(key)
|
||||
if dir == req.Directory {
|
||||
if req.StartFromFileName != "" && name <= req.StartFromFileName {
|
||||
continue
|
||||
}
|
||||
if req.Prefix != "" && !strings.HasPrefix(name, req.Prefix) {
|
||||
continue
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
// Clone entries while still holding the lock.
|
||||
type namedEntry struct {
|
||||
name string
|
||||
entry *filer_pb.Entry
|
||||
}
|
||||
snapshot := make([]namedEntry, 0, len(names))
|
||||
for _, name := range names {
|
||||
if req.Limit > 0 && uint32(len(snapshot)) >= req.Limit {
|
||||
break
|
||||
}
|
||||
snapshot = append(snapshot, namedEntry{
|
||||
name: name,
|
||||
entry: proto.Clone(s.entries[s.key(req.Directory, name)]).(*filer_pb.Entry),
|
||||
})
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
// Stream responses without holding any lock.
|
||||
for _, ne := range snapshot {
|
||||
if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: ne.entry}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *testFilerServer) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.entries[s.key(req.Directory, req.Entry.Name)] = proto.Clone(req.Entry).(*filer_pb.Entry)
|
||||
return &filer_pb.CreateEntryResponse{}, nil
|
||||
}
|
||||
|
||||
func (s *testFilerServer) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
k := s.key(req.Directory, req.Name)
|
||||
if _, found := s.entries[k]; !found {
|
||||
return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
|
||||
}
|
||||
delete(s.entries, k)
|
||||
if req.IsRecursive {
|
||||
// Delete all descendants: any entry whose directory starts with
|
||||
// the deleted path (handles nested subdirectories).
|
||||
deletedPath := req.Directory + "/" + req.Name
|
||||
for key := range s.entries {
|
||||
dir, _ := s.splitKey(key)
|
||||
if dir == deletedPath || strings.HasPrefix(dir, deletedPath+"/") {
|
||||
delete(s.entries, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
return &filer_pb.DeleteEntryResponse{}, nil
|
||||
}
|
||||
|
||||
// startTestFiler starts an in-memory filer gRPC server and returns a client.
|
||||
func startTestFiler(t *testing.T) (*testFilerServer, filer_pb.SeaweedFilerClient) {
|
||||
t.Helper()
|
||||
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
|
||||
server := newTestFilerServer()
|
||||
grpcServer := pb.NewGrpcServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(grpcServer, server)
|
||||
go func() { _ = grpcServer.Serve(lis) }()
|
||||
|
||||
t.Cleanup(func() {
|
||||
grpcServer.Stop()
|
||||
_ = lis.Close()
|
||||
})
|
||||
|
||||
host, portStr, err := net.SplitHostPort(lis.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("split host port: %v", err)
|
||||
}
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
t.Fatalf("parse port: %v", err)
|
||||
}
|
||||
addr := pb.NewServerAddress(host, 1, port)
|
||||
|
||||
conn, err := pb.GrpcDial(context.Background(), addr.ToGrpcAddress(), false, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
t.Fatalf("dial: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = conn.Close() })
|
||||
|
||||
return server, filer_pb.NewSeaweedFilerClient(conn)
|
||||
}
|
||||
|
||||
// Helper to create a version ID from a timestamp.
|
||||
func testVersionId(ts time.Time) string {
|
||||
inverted := math.MaxInt64 - ts.UnixNano()
|
||||
return fmt.Sprintf("%016x", inverted) + "0000000000000000"
|
||||
}
|
||||
|
||||
func TestIntegration_ListExpiredObjectsByRules(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "test-bucket"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
|
||||
now := time.Now()
|
||||
old := now.Add(-60 * 24 * time.Hour) // 60 days ago
|
||||
recent := now.Add(-5 * 24 * time.Hour) // 5 days ago
|
||||
|
||||
// Create bucket directory.
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
|
||||
// Create objects.
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "old-file.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 1024},
|
||||
})
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "recent-file.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: recent.Unix(), FileSize: 1024},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "expire-30d", Status: "Enabled",
|
||||
ExpirationDays: 30,
|
||||
}}
|
||||
|
||||
expired, scanned, err := listExpiredObjectsByRules(context.Background(), client, bucketsPath, bucket, rules, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("listExpiredObjectsByRules: %v", err)
|
||||
}
|
||||
|
||||
if scanned != 2 {
|
||||
t.Errorf("expected 2 scanned, got %d", scanned)
|
||||
}
|
||||
if len(expired) != 1 {
|
||||
t.Fatalf("expected 1 expired, got %d", len(expired))
|
||||
}
|
||||
if expired[0].name != "old-file.txt" {
|
||||
t.Errorf("expected old-file.txt expired, got %s", expired[0].name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegration_ListExpiredObjectsByRules_TagFilter(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "tag-bucket"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
|
||||
old := time.Now().Add(-60 * 24 * time.Hour)
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
|
||||
// Object with matching tag.
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "tagged.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 100},
|
||||
Extended: map[string][]byte{"X-Amz-Tagging-env": []byte("dev")},
|
||||
})
|
||||
// Object without tag.
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "untagged.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 100},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "tag-expire", Status: "Enabled",
|
||||
ExpirationDays: 30,
|
||||
FilterTags: map[string]string{"env": "dev"},
|
||||
}}
|
||||
|
||||
expired, _, err := listExpiredObjectsByRules(context.Background(), client, bucketsPath, bucket, rules, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("listExpiredObjectsByRules: %v", err)
|
||||
}
|
||||
|
||||
if len(expired) != 1 {
|
||||
t.Fatalf("expected 1 expired (tagged only), got %d", len(expired))
|
||||
}
|
||||
if expired[0].name != "tagged.txt" {
|
||||
t.Errorf("expected tagged.txt, got %s", expired[0].name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegration_ProcessVersionsDirectory(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "versioned-bucket"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
versionsDir := bucketDir + "/key.versions"
|
||||
|
||||
now := time.Now()
|
||||
t1 := now.Add(-90 * 24 * time.Hour) // oldest
|
||||
t2 := now.Add(-60 * 24 * time.Hour)
|
||||
t3 := now.Add(-1 * 24 * time.Hour) // newest (latest)
|
||||
|
||||
vid1 := testVersionId(t1)
|
||||
vid2 := testVersionId(t2)
|
||||
vid3 := testVersionId(t3)
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "key.versions", IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(vid3),
|
||||
},
|
||||
})
|
||||
|
||||
// Three versions: vid3 (latest), vid2 (noncurrent), vid1 (noncurrent)
|
||||
server.putEntry(versionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vid3,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: t3.Unix(), FileSize: 100},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(vid3),
|
||||
},
|
||||
})
|
||||
server.putEntry(versionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vid2,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: t2.Unix(), FileSize: 100},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(vid2),
|
||||
},
|
||||
})
|
||||
server.putEntry(versionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vid1,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: t1.Unix(), FileSize: 100},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(vid1),
|
||||
},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "noncurrent-30d", Status: "Enabled",
|
||||
NoncurrentVersionExpirationDays: 30,
|
||||
}}
|
||||
|
||||
expired, scanned, err := processVersionsDirectory(
|
||||
context.Background(), client, versionsDir, bucketDir,
|
||||
rules, now, false, 100,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("processVersionsDirectory: %v", err)
|
||||
}
|
||||
|
||||
// vid3 is latest (not expired). vid2 became noncurrent when vid3 was created
|
||||
// (1 day ago), so vid2 is NOT old enough (< 30 days noncurrent).
|
||||
// vid1 became noncurrent when vid2 was created (60 days ago), so vid1 IS expired.
|
||||
if scanned != 2 {
|
||||
t.Errorf("expected 2 scanned (non-current versions), got %d", scanned)
|
||||
}
|
||||
if len(expired) != 1 {
|
||||
t.Fatalf("expected 1 expired (only vid1), got %d", len(expired))
|
||||
}
|
||||
if expired[0].name != "v_"+vid1 {
|
||||
t.Errorf("expected v_%s expired, got %s", vid1, expired[0].name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegration_ProcessVersionsDirectory_NewerNoncurrentVersions(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "keep-n-bucket"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
versionsDir := bucketDir + "/obj.versions"
|
||||
|
||||
now := time.Now()
|
||||
// Create 5 versions, all old enough to expire by days alone.
|
||||
versions := make([]time.Time, 5)
|
||||
vids := make([]string, 5)
|
||||
for i := 0; i < 5; i++ {
|
||||
versions[i] = now.Add(time.Duration(-(90 - i*10)) * 24 * time.Hour)
|
||||
vids[i] = testVersionId(versions[i])
|
||||
}
|
||||
// vids[4] is newest (latest), vids[0] is oldest
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "obj.versions", IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(vids[4]),
|
||||
},
|
||||
})
|
||||
|
||||
for i, vid := range vids {
|
||||
server.putEntry(versionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vid,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: versions[i].Unix(), FileSize: 100},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(vid),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "keep-2", Status: "Enabled",
|
||||
NoncurrentVersionExpirationDays: 7,
|
||||
NewerNoncurrentVersions: 2,
|
||||
}}
|
||||
|
||||
expired, _, err := processVersionsDirectory(
|
||||
context.Background(), client, versionsDir, bucketDir,
|
||||
rules, now, false, 100,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("processVersionsDirectory: %v", err)
|
||||
}
|
||||
|
||||
// 4 noncurrent versions (vids[0..3]). Keep newest 2 (vids[3], vids[2]).
|
||||
// Expire vids[1] and vids[0].
|
||||
if len(expired) != 2 {
|
||||
t.Fatalf("expected 2 expired (keep 2 newest noncurrent), got %d", len(expired))
|
||||
}
|
||||
expiredNames := map[string]bool{}
|
||||
for _, e := range expired {
|
||||
expiredNames[e.name] = true
|
||||
}
|
||||
if !expiredNames["v_"+vids[0]] {
|
||||
t.Errorf("expected vids[0] (oldest) to be expired")
|
||||
}
|
||||
if !expiredNames["v_"+vids[1]] {
|
||||
t.Errorf("expected vids[1] to be expired")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegration_AbortMPUsByRules(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "mpu-bucket"
|
||||
uploadsDir := bucketsPath + "/" + bucket + "/.uploads"
|
||||
|
||||
now := time.Now()
|
||||
old := now.Add(-10 * 24 * time.Hour)
|
||||
recent := now.Add(-2 * 24 * time.Hour)
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
server.putEntry(bucketsPath+"/"+bucket, &filer_pb.Entry{Name: ".uploads", IsDirectory: true})
|
||||
|
||||
// Old upload under logs/ prefix.
|
||||
server.putEntry(uploadsDir, &filer_pb.Entry{
|
||||
Name: "logs_upload1", IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{Crtime: old.Unix()},
|
||||
})
|
||||
// Recent upload under logs/ prefix.
|
||||
server.putEntry(uploadsDir, &filer_pb.Entry{
|
||||
Name: "logs_upload2", IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{Crtime: recent.Unix()},
|
||||
})
|
||||
// Old upload under data/ prefix (should not match logs/ rule).
|
||||
server.putEntry(uploadsDir, &filer_pb.Entry{
|
||||
Name: "data_upload1", IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{Crtime: old.Unix()},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "abort-logs", Status: "Enabled",
|
||||
Prefix: "logs",
|
||||
AbortMPUDaysAfterInitiation: 7,
|
||||
}}
|
||||
|
||||
aborted, errs, err := abortMPUsByRules(context.Background(), client, bucketsPath, bucket, rules, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("abortMPUsByRules: %v", err)
|
||||
}
|
||||
if errs != 0 {
|
||||
t.Errorf("expected 0 errors, got %d", errs)
|
||||
}
|
||||
|
||||
// Only logs_upload1 should be aborted (old + matches prefix).
|
||||
// logs_upload2 is too recent, data_upload1 doesn't match prefix.
|
||||
if aborted != 1 {
|
||||
t.Errorf("expected 1 aborted, got %d", aborted)
|
||||
}
|
||||
|
||||
// Verify the correct upload was removed.
|
||||
if server.hasEntry(uploadsDir, "logs_upload1") {
|
||||
t.Error("logs_upload1 should have been removed")
|
||||
}
|
||||
if !server.hasEntry(uploadsDir, "logs_upload2") {
|
||||
t.Error("logs_upload2 should still exist")
|
||||
}
|
||||
if !server.hasEntry(uploadsDir, "data_upload1") {
|
||||
t.Error("data_upload1 should still exist (wrong prefix)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegration_DeleteExpiredObjects(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "delete-bucket"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
|
||||
now := time.Now()
|
||||
old := now.Add(-60 * 24 * time.Hour)
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "to-delete.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 100},
|
||||
})
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "to-keep.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: now.Unix(), FileSize: 100},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "expire", Status: "Enabled",
|
||||
ExpirationDays: 30,
|
||||
}}
|
||||
|
||||
expired, _, err := listExpiredObjectsByRules(context.Background(), client, bucketsPath, bucket, rules, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
|
||||
// Actually delete them.
|
||||
deleted, errs, err := deleteExpiredObjects(context.Background(), client, expired)
|
||||
if err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if deleted != 1 || errs != 0 {
|
||||
t.Errorf("expected 1 deleted 0 errors, got %d deleted %d errors", deleted, errs)
|
||||
}
|
||||
|
||||
if server.hasEntry(bucketDir, "to-delete.txt") {
|
||||
t.Error("to-delete.txt should have been removed")
|
||||
}
|
||||
if !server.hasEntry(bucketDir, "to-keep.txt") {
|
||||
t.Error("to-keep.txt should still exist")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIntegration_VersionedBucket_ExpirationDays verifies that Expiration.Days
|
||||
// rules correctly detect and delete the latest version in a versioned bucket
|
||||
// where all data lives in .versions/ directories (issue #8757).
|
||||
func TestIntegration_VersionedBucket_ExpirationDays(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "versioned-expire"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
|
||||
now := time.Now()
|
||||
old := now.Add(-60 * 24 * time.Hour) // 60 days ago — should expire
|
||||
recent := now.Add(-5 * 24 * time.Hour) // 5 days ago — should NOT expire
|
||||
|
||||
vidOld := testVersionId(old)
|
||||
vidRecent := testVersionId(recent)
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
|
||||
// --- Single-version object (old, should expire) ---
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "old-file.txt" + s3_constants.VersionsFolder, IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(vidOld),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + vidOld),
|
||||
s3_constants.ExtLatestVersionMtimeKey: []byte(strconv.FormatInt(old.Unix(), 10)),
|
||||
s3_constants.ExtLatestVersionSizeKey: []byte("3400000000"),
|
||||
s3_constants.ExtLatestVersionIsDeleteMarker: []byte("false"),
|
||||
},
|
||||
})
|
||||
oldVersionsDir := bucketDir + "/old-file.txt" + s3_constants.VersionsFolder
|
||||
server.putEntry(oldVersionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vidOld,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 3400000000},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(vidOld),
|
||||
},
|
||||
})
|
||||
|
||||
// --- Single-version object (recent, should NOT expire) ---
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "recent-file.txt" + s3_constants.VersionsFolder, IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(vidRecent),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + vidRecent),
|
||||
s3_constants.ExtLatestVersionMtimeKey: []byte(strconv.FormatInt(recent.Unix(), 10)),
|
||||
s3_constants.ExtLatestVersionSizeKey: []byte("3400000000"),
|
||||
s3_constants.ExtLatestVersionIsDeleteMarker: []byte("false"),
|
||||
},
|
||||
})
|
||||
recentVersionsDir := bucketDir + "/recent-file.txt" + s3_constants.VersionsFolder
|
||||
server.putEntry(recentVersionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vidRecent,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: recent.Unix(), FileSize: 3400000000},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(vidRecent),
|
||||
},
|
||||
})
|
||||
|
||||
// --- Object with delete marker as latest (should NOT be expired by Expiration.Days) ---
|
||||
vidMarker := testVersionId(old)
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "deleted-obj.txt" + s3_constants.VersionsFolder, IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(vidMarker),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + vidMarker),
|
||||
s3_constants.ExtLatestVersionMtimeKey: []byte(strconv.FormatInt(old.Unix(), 10)),
|
||||
s3_constants.ExtLatestVersionIsDeleteMarker: []byte("true"),
|
||||
},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "expire-30d", Status: "Enabled",
|
||||
ExpirationDays: 30,
|
||||
}}
|
||||
|
||||
expired, scanned, err := listExpiredObjectsByRules(context.Background(), client, bucketsPath, bucket, rules, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("listExpiredObjectsByRules: %v", err)
|
||||
}
|
||||
|
||||
// Only old-file.txt's latest version should be expired.
|
||||
// recent-file.txt is too young; deleted-obj.txt is a delete marker.
|
||||
if len(expired) != 1 {
|
||||
t.Fatalf("expected 1 expired, got %d: %+v", len(expired), expired)
|
||||
}
|
||||
if expired[0].dir != oldVersionsDir {
|
||||
t.Errorf("expected dir=%s, got %s", oldVersionsDir, expired[0].dir)
|
||||
}
|
||||
if expired[0].name != "v_"+vidOld {
|
||||
t.Errorf("expected name=v_%s, got %s", vidOld, expired[0].name)
|
||||
}
|
||||
// The old-file.txt latest version should count as scanned.
|
||||
if scanned < 1 {
|
||||
t.Errorf("expected at least 1 scanned, got %d", scanned)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIntegration_VersionedBucket_ExpirationDays_DeleteAndCleanup verifies
|
||||
// end-to-end deletion and .versions directory cleanup for a single-version
|
||||
// versioned object expired by Expiration.Days.
|
||||
func TestIntegration_VersionedBucket_ExpirationDays_DeleteAndCleanup(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "versioned-cleanup"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
|
||||
now := time.Now()
|
||||
old := now.Add(-60 * 24 * time.Hour)
|
||||
vidOld := testVersionId(old)
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
|
||||
// Single-version object that should expire.
|
||||
versionsDir := bucketDir + "/data.bin" + s3_constants.VersionsFolder
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "data.bin" + s3_constants.VersionsFolder, IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(vidOld),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + vidOld),
|
||||
s3_constants.ExtLatestVersionMtimeKey: []byte(strconv.FormatInt(old.Unix(), 10)),
|
||||
s3_constants.ExtLatestVersionSizeKey: []byte("1024"),
|
||||
s3_constants.ExtLatestVersionIsDeleteMarker: []byte("false"),
|
||||
},
|
||||
})
|
||||
server.putEntry(versionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vidOld,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 1024},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte(vidOld),
|
||||
},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "expire-30d", Status: "Enabled",
|
||||
ExpirationDays: 30,
|
||||
}}
|
||||
|
||||
// Step 1: Detect expired.
|
||||
expired, _, err := listExpiredObjectsByRules(context.Background(), client, bucketsPath, bucket, rules, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(expired) != 1 {
|
||||
t.Fatalf("expected 1 expired, got %d", len(expired))
|
||||
}
|
||||
|
||||
// Step 2: Delete the expired version file.
|
||||
deleted, errs, delErr := deleteExpiredObjects(context.Background(), client, expired)
|
||||
if delErr != nil {
|
||||
t.Fatalf("delete: %v", delErr)
|
||||
}
|
||||
if deleted != 1 || errs != 0 {
|
||||
t.Errorf("expected 1 deleted 0 errors, got %d deleted %d errors", deleted, errs)
|
||||
}
|
||||
|
||||
// Version file should be gone.
|
||||
if server.hasEntry(versionsDir, "v_"+vidOld) {
|
||||
t.Error("version file should have been removed")
|
||||
}
|
||||
|
||||
// Step 3: Cleanup empty .versions directory.
|
||||
cleaned := cleanupEmptyVersionsDirectories(context.Background(), client, expired)
|
||||
if cleaned != 1 {
|
||||
t.Errorf("expected 1 directory cleaned, got %d", cleaned)
|
||||
}
|
||||
|
||||
// The .versions directory itself should be gone.
|
||||
if server.hasEntry(bucketDir, "data.bin"+s3_constants.VersionsFolder) {
|
||||
t.Error(".versions directory should have been removed after cleanup")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIntegration_VersionedBucket_MultiVersion_ExpirationDays verifies that
|
||||
// when a multi-version object's latest version expires, only the latest
|
||||
// version is deleted and noncurrent versions remain.
|
||||
func TestIntegration_VersionedBucket_MultiVersion_ExpirationDays(t *testing.T) {
|
||||
server, client := startTestFiler(t)
|
||||
bucketsPath := "/buckets"
|
||||
bucket := "versioned-multi"
|
||||
bucketDir := bucketsPath + "/" + bucket
|
||||
|
||||
now := time.Now()
|
||||
tOld := now.Add(-60 * 24 * time.Hour)
|
||||
tNoncurrent := now.Add(-90 * 24 * time.Hour)
|
||||
vidLatest := testVersionId(tOld)
|
||||
vidNoncurrent := testVersionId(tNoncurrent)
|
||||
|
||||
server.putEntry(bucketsPath, &filer_pb.Entry{Name: bucket, IsDirectory: true})
|
||||
|
||||
versionsDir := bucketDir + "/multi.txt" + s3_constants.VersionsFolder
|
||||
server.putEntry(bucketDir, &filer_pb.Entry{
|
||||
Name: "multi.txt" + s3_constants.VersionsFolder, IsDirectory: true,
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(vidLatest),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + vidLatest),
|
||||
s3_constants.ExtLatestVersionMtimeKey: []byte(strconv.FormatInt(tOld.Unix(), 10)),
|
||||
s3_constants.ExtLatestVersionSizeKey: []byte("500"),
|
||||
s3_constants.ExtLatestVersionIsDeleteMarker: []byte("false"),
|
||||
},
|
||||
})
|
||||
server.putEntry(versionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vidLatest,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: tOld.Unix(), FileSize: 500},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(vidLatest)},
|
||||
})
|
||||
server.putEntry(versionsDir, &filer_pb.Entry{
|
||||
Name: "v_" + vidNoncurrent,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: tNoncurrent.Unix(), FileSize: 500},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(vidNoncurrent)},
|
||||
})
|
||||
|
||||
rules := []s3lifecycle.Rule{{
|
||||
ID: "expire-30d", Status: "Enabled",
|
||||
ExpirationDays: 30,
|
||||
}}
|
||||
|
||||
expired, _, err := listExpiredObjectsByRules(context.Background(), client, bucketsPath, bucket, rules, 100)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
// Only the latest version should be detected as expired.
|
||||
if len(expired) != 1 {
|
||||
t.Fatalf("expected 1 expired (latest only), got %d", len(expired))
|
||||
}
|
||||
if expired[0].name != "v_"+vidLatest {
|
||||
t.Errorf("expected latest version expired, got %s", expired[0].name)
|
||||
}
|
||||
|
||||
// Delete it.
|
||||
deleted, errs, delErr := deleteExpiredObjects(context.Background(), client, expired)
|
||||
if delErr != nil {
|
||||
t.Fatalf("delete: %v", delErr)
|
||||
}
|
||||
if deleted != 1 || errs != 0 {
|
||||
t.Errorf("expected 1 deleted 0 errors, got %d deleted %d errors", deleted, errs)
|
||||
}
|
||||
|
||||
// Noncurrent version should still exist.
|
||||
if !server.hasEntry(versionsDir, "v_"+vidNoncurrent) {
|
||||
t.Error("noncurrent version should still exist")
|
||||
}
|
||||
|
||||
// .versions directory should NOT be cleaned up (not empty).
|
||||
cleaned := cleanupEmptyVersionsDirectories(context.Background(), client, expired)
|
||||
if cleaned != 0 {
|
||||
t.Errorf("expected 0 directories cleaned (not empty), got %d", cleaned)
|
||||
}
|
||||
if !server.hasEntry(bucketDir, "multi.txt"+s3_constants.VersionsFolder) {
|
||||
t.Error(".versions directory should still exist (has noncurrent version)")
|
||||
}
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
// lifecycleConfig mirrors the XML structure just enough to parse rules.
|
||||
// We define a minimal local struct to avoid importing the s3api package
|
||||
// (which would create a circular dependency if s3api ever imports the worker).
|
||||
type lifecycleConfig struct {
|
||||
XMLName xml.Name `xml:"LifecycleConfiguration"`
|
||||
Rules []lifecycleConfigRule `xml:"Rule"`
|
||||
}
|
||||
|
||||
type lifecycleConfigRule struct {
|
||||
ID string `xml:"ID"`
|
||||
Status string `xml:"Status"`
|
||||
Filter lifecycleFilter `xml:"Filter"`
|
||||
Prefix string `xml:"Prefix"`
|
||||
Expiration lifecycleExpiration `xml:"Expiration"`
|
||||
NoncurrentVersionExpiration noncurrentVersionExpiration `xml:"NoncurrentVersionExpiration"`
|
||||
AbortIncompleteMultipartUpload abortMPU `xml:"AbortIncompleteMultipartUpload"`
|
||||
}
|
||||
|
||||
type lifecycleFilter struct {
|
||||
Prefix string `xml:"Prefix"`
|
||||
Tag lifecycleTag `xml:"Tag"`
|
||||
And lifecycleAnd `xml:"And"`
|
||||
ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan"`
|
||||
ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan"`
|
||||
}
|
||||
|
||||
type lifecycleAnd struct {
|
||||
Prefix string `xml:"Prefix"`
|
||||
Tags []lifecycleTag `xml:"Tag"`
|
||||
ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan"`
|
||||
ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan"`
|
||||
}
|
||||
|
||||
type lifecycleTag struct {
|
||||
Key string `xml:"Key"`
|
||||
Value string `xml:"Value"`
|
||||
}
|
||||
|
||||
type lifecycleExpiration struct {
|
||||
Days int `xml:"Days"`
|
||||
Date string `xml:"Date"`
|
||||
ExpiredObjectDeleteMarker bool `xml:"ExpiredObjectDeleteMarker"`
|
||||
}
|
||||
|
||||
type noncurrentVersionExpiration struct {
|
||||
NoncurrentDays int `xml:"NoncurrentDays"`
|
||||
NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions"`
|
||||
}
|
||||
|
||||
type abortMPU struct {
|
||||
DaysAfterInitiation int `xml:"DaysAfterInitiation"`
|
||||
}
|
||||
|
||||
// errMalformedLifecycleXML indicates the lifecycle XML exists but could not be parsed.
|
||||
// Callers should fail closed (not fall back to TTL) to avoid broader deletions.
|
||||
var errMalformedLifecycleXML = errors.New("malformed lifecycle XML")
|
||||
|
||||
// loadLifecycleRulesFromBucket reads the lifecycle XML from a bucket's
|
||||
// metadata and converts it to evaluator-friendly rules.
|
||||
//
|
||||
// Returns:
|
||||
// - (rules, nil) when lifecycle XML is configured and parseable
|
||||
// - (nil, nil) when no lifecycle XML is configured (caller should use TTL fallback)
|
||||
// - (nil, errMalformedLifecycleXML) when XML exists but is malformed (fail closed)
|
||||
// - (nil, err) for transient filer errors (caller should use TTL fallback with warning)
|
||||
func loadLifecycleRulesFromBucket(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
bucketsPath, bucket string,
|
||||
) ([]s3lifecycle.Rule, error) {
|
||||
bucketDir := bucketsPath
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: bucketDir,
|
||||
Name: bucket,
|
||||
})
|
||||
if err != nil {
|
||||
// Transient filer error — not the same as malformed XML.
|
||||
return nil, fmt.Errorf("lookup bucket %s: %w", bucket, err)
|
||||
}
|
||||
if resp.Entry == nil || resp.Entry.Extended == nil {
|
||||
return nil, nil
|
||||
}
|
||||
xmlData := resp.Entry.Extended[lifecycleXMLKey]
|
||||
if len(xmlData) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
rules, parseErr := parseLifecycleXML(xmlData)
|
||||
if parseErr != nil {
|
||||
return nil, fmt.Errorf("%w: bucket %s: %v", errMalformedLifecycleXML, bucket, parseErr)
|
||||
}
|
||||
// Return non-nil empty slice when XML was present but yielded no rules
|
||||
// (e.g., all rules disabled). This lets callers distinguish "no XML" (nil)
|
||||
// from "XML present, no effective rules" (empty slice).
|
||||
if rules == nil {
|
||||
rules = []s3lifecycle.Rule{}
|
||||
}
|
||||
return rules, nil
|
||||
}
|
||||
|
||||
// parseLifecycleXML parses lifecycle configuration XML and converts it
|
||||
// to evaluator-friendly rules.
|
||||
func parseLifecycleXML(data []byte) ([]s3lifecycle.Rule, error) {
|
||||
var config lifecycleConfig
|
||||
if err := xml.NewDecoder(bytes.NewReader(data)).Decode(&config); err != nil {
|
||||
return nil, fmt.Errorf("decode lifecycle XML: %w", err)
|
||||
}
|
||||
|
||||
var rules []s3lifecycle.Rule
|
||||
for _, r := range config.Rules {
|
||||
rule := s3lifecycle.Rule{
|
||||
ID: r.ID,
|
||||
Status: r.Status,
|
||||
}
|
||||
|
||||
// Resolve prefix: Filter.And.Prefix > Filter.Prefix > Rule.Prefix
|
||||
switch {
|
||||
case r.Filter.And.Prefix != "" || len(r.Filter.And.Tags) > 0 ||
|
||||
r.Filter.And.ObjectSizeGreaterThan > 0 || r.Filter.And.ObjectSizeLessThan > 0:
|
||||
rule.Prefix = r.Filter.And.Prefix
|
||||
rule.FilterTags = tagsToMap(r.Filter.And.Tags)
|
||||
rule.FilterSizeGreaterThan = r.Filter.And.ObjectSizeGreaterThan
|
||||
rule.FilterSizeLessThan = r.Filter.And.ObjectSizeLessThan
|
||||
case r.Filter.Tag.Key != "":
|
||||
rule.Prefix = r.Filter.Prefix
|
||||
rule.FilterTags = map[string]string{r.Filter.Tag.Key: r.Filter.Tag.Value}
|
||||
rule.FilterSizeGreaterThan = r.Filter.ObjectSizeGreaterThan
|
||||
rule.FilterSizeLessThan = r.Filter.ObjectSizeLessThan
|
||||
default:
|
||||
if r.Filter.Prefix != "" {
|
||||
rule.Prefix = r.Filter.Prefix
|
||||
} else {
|
||||
rule.Prefix = r.Prefix
|
||||
}
|
||||
rule.FilterSizeGreaterThan = r.Filter.ObjectSizeGreaterThan
|
||||
rule.FilterSizeLessThan = r.Filter.ObjectSizeLessThan
|
||||
}
|
||||
|
||||
rule.ExpirationDays = r.Expiration.Days
|
||||
rule.ExpiredObjectDeleteMarker = r.Expiration.ExpiredObjectDeleteMarker
|
||||
rule.NoncurrentVersionExpirationDays = r.NoncurrentVersionExpiration.NoncurrentDays
|
||||
rule.NewerNoncurrentVersions = r.NoncurrentVersionExpiration.NewerNoncurrentVersions
|
||||
rule.AbortMPUDaysAfterInitiation = r.AbortIncompleteMultipartUpload.DaysAfterInitiation
|
||||
|
||||
// Parse Date if present.
|
||||
if r.Expiration.Date != "" {
|
||||
// Date may be RFC3339 or ISO 8601 date-only.
|
||||
parsed, parseErr := parseExpirationDate(r.Expiration.Date)
|
||||
if parseErr != nil {
|
||||
glog.V(1).Infof("s3_lifecycle: skipping rule %s: invalid expiration date %q: %v", r.ID, r.Expiration.Date, parseErr)
|
||||
continue
|
||||
}
|
||||
rule.ExpirationDate = parsed
|
||||
}
|
||||
|
||||
rules = append(rules, rule)
|
||||
}
|
||||
return rules, nil
|
||||
}
|
||||
|
||||
func tagsToMap(tags []lifecycleTag) map[string]string {
|
||||
if len(tags) == 0 {
|
||||
return nil
|
||||
}
|
||||
m := make(map[string]string, len(tags))
|
||||
for _, t := range tags {
|
||||
m[t.Key] = t.Value
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func parseExpirationDate(s string) (time.Time, error) {
|
||||
// Try RFC3339 first, then ISO 8601 date-only.
|
||||
formats := []string{
|
||||
"2006-01-02T15:04:05Z07:00",
|
||||
"2006-01-02",
|
||||
}
|
||||
for _, f := range formats {
|
||||
t, err := time.Parse(f, s)
|
||||
if err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unrecognized date format: %s", s)
|
||||
}
|
||||
@@ -1,256 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseLifecycleXML_CompleteConfig(t *testing.T) {
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>rotation</ID>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<Status>Enabled</Status>
|
||||
<Expiration><Days>30</Days></Expiration>
|
||||
<NoncurrentVersionExpiration>
|
||||
<NoncurrentDays>7</NoncurrentDays>
|
||||
<NewerNoncurrentVersions>2</NewerNoncurrentVersions>
|
||||
</NoncurrentVersionExpiration>
|
||||
<AbortIncompleteMultipartUpload>
|
||||
<DaysAfterInitiation>3</DaysAfterInitiation>
|
||||
</AbortIncompleteMultipartUpload>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
if len(rules) != 1 {
|
||||
t.Fatalf("expected 1 rule, got %d", len(rules))
|
||||
}
|
||||
|
||||
r := rules[0]
|
||||
if r.ID != "rotation" {
|
||||
t.Errorf("expected ID 'rotation', got %q", r.ID)
|
||||
}
|
||||
if r.Status != "Enabled" {
|
||||
t.Errorf("expected Status 'Enabled', got %q", r.Status)
|
||||
}
|
||||
if r.ExpirationDays != 30 {
|
||||
t.Errorf("expected ExpirationDays=30, got %d", r.ExpirationDays)
|
||||
}
|
||||
if r.NoncurrentVersionExpirationDays != 7 {
|
||||
t.Errorf("expected NoncurrentVersionExpirationDays=7, got %d", r.NoncurrentVersionExpirationDays)
|
||||
}
|
||||
if r.NewerNoncurrentVersions != 2 {
|
||||
t.Errorf("expected NewerNoncurrentVersions=2, got %d", r.NewerNoncurrentVersions)
|
||||
}
|
||||
if r.AbortMPUDaysAfterInitiation != 3 {
|
||||
t.Errorf("expected AbortMPUDaysAfterInitiation=3, got %d", r.AbortMPUDaysAfterInitiation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLifecycleXML_PrefixFilter(t *testing.T) {
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>logs</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>logs/</Prefix></Filter>
|
||||
<Expiration><Days>7</Days></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
if len(rules) != 1 {
|
||||
t.Fatalf("expected 1 rule, got %d", len(rules))
|
||||
}
|
||||
if rules[0].Prefix != "logs/" {
|
||||
t.Errorf("expected Prefix='logs/', got %q", rules[0].Prefix)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLifecycleXML_LegacyPrefix(t *testing.T) {
|
||||
// Old-style <Prefix> at rule level instead of inside <Filter>.
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>old</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Prefix>archive/</Prefix>
|
||||
<Expiration><Days>90</Days></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
if len(rules) != 1 {
|
||||
t.Fatalf("expected 1 rule, got %d", len(rules))
|
||||
}
|
||||
if rules[0].Prefix != "archive/" {
|
||||
t.Errorf("expected Prefix='archive/', got %q", rules[0].Prefix)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLifecycleXML_TagFilter(t *testing.T) {
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>tag-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<Tag><Key>env</Key><Value>dev</Value></Tag>
|
||||
</Filter>
|
||||
<Expiration><Days>1</Days></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
if len(rules) != 1 {
|
||||
t.Fatalf("expected 1 rule, got %d", len(rules))
|
||||
}
|
||||
r := rules[0]
|
||||
if len(r.FilterTags) != 1 || r.FilterTags["env"] != "dev" {
|
||||
t.Errorf("expected FilterTags={env:dev}, got %v", r.FilterTags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLifecycleXML_AndFilter(t *testing.T) {
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>and-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter>
|
||||
<And>
|
||||
<Prefix>data/</Prefix>
|
||||
<Tag><Key>env</Key><Value>staging</Value></Tag>
|
||||
<ObjectSizeGreaterThan>1024</ObjectSizeGreaterThan>
|
||||
</And>
|
||||
</Filter>
|
||||
<Expiration><Days>14</Days></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
if len(rules) != 1 {
|
||||
t.Fatalf("expected 1 rule, got %d", len(rules))
|
||||
}
|
||||
r := rules[0]
|
||||
if r.Prefix != "data/" {
|
||||
t.Errorf("expected Prefix='data/', got %q", r.Prefix)
|
||||
}
|
||||
if r.FilterTags["env"] != "staging" {
|
||||
t.Errorf("expected tag env=staging, got %v", r.FilterTags)
|
||||
}
|
||||
if r.FilterSizeGreaterThan != 1024 {
|
||||
t.Errorf("expected FilterSizeGreaterThan=1024, got %d", r.FilterSizeGreaterThan)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLifecycleXML_ExpirationDate(t *testing.T) {
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>date-rule</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<Expiration><Date>2026-06-01T00:00:00Z</Date></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
expected := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC)
|
||||
if !rules[0].ExpirationDate.Equal(expected) {
|
||||
t.Errorf("expected ExpirationDate=%v, got %v", expected, rules[0].ExpirationDate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLifecycleXML_ExpiredObjectDeleteMarker(t *testing.T) {
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>marker-cleanup</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
if !rules[0].ExpiredObjectDeleteMarker {
|
||||
t.Error("expected ExpiredObjectDeleteMarker=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLifecycleXML_MultipleRules(t *testing.T) {
|
||||
xml := []byte(`<LifecycleConfiguration>
|
||||
<Rule>
|
||||
<ID>rule1</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix>logs/</Prefix></Filter>
|
||||
<Expiration><Days>7</Days></Expiration>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>rule2</ID>
|
||||
<Status>Disabled</Status>
|
||||
<Filter><Prefix>temp/</Prefix></Filter>
|
||||
<Expiration><Days>1</Days></Expiration>
|
||||
</Rule>
|
||||
<Rule>
|
||||
<ID>rule3</ID>
|
||||
<Status>Enabled</Status>
|
||||
<Filter><Prefix></Prefix></Filter>
|
||||
<Expiration><Days>365</Days></Expiration>
|
||||
</Rule>
|
||||
</LifecycleConfiguration>`)
|
||||
|
||||
rules, err := parseLifecycleXML(xml)
|
||||
if err != nil {
|
||||
t.Fatalf("parseLifecycleXML: %v", err)
|
||||
}
|
||||
if len(rules) != 3 {
|
||||
t.Fatalf("expected 3 rules, got %d", len(rules))
|
||||
}
|
||||
if rules[1].Status != "Disabled" {
|
||||
t.Errorf("expected rule2 Status=Disabled, got %q", rules[1].Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExpirationDate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want time.Time
|
||||
wantErr bool
|
||||
}{
|
||||
{"rfc3339_utc", "2026-06-01T00:00:00Z", time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC), false},
|
||||
{"rfc3339_offset", "2026-06-01T00:00:00+05:00", time.Date(2026, 6, 1, 0, 0, 0, 0, time.FixedZone("", 5*3600)), false},
|
||||
{"date_only", "2026-06-01", time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC), false},
|
||||
{"invalid", "not-a-date", time.Time{}, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseExpirationDate(tt.input)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseExpirationDate(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !tt.wantErr && !got.Equal(tt.want) {
|
||||
t.Errorf("parseExpirationDate(%q) = %v, want %v", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
)
|
||||
|
||||
// makeVersionId creates a new-format version ID from a timestamp.
|
||||
func makeVersionId(t time.Time) string {
|
||||
inverted := math.MaxInt64 - t.UnixNano()
|
||||
return fmt.Sprintf("%016x", inverted) + "0000000000000000"
|
||||
}
|
||||
|
||||
func TestSortVersionsByVersionId(t *testing.T) {
|
||||
t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
t2 := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC)
|
||||
t3 := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
vid1 := makeVersionId(t1)
|
||||
vid2 := makeVersionId(t2)
|
||||
vid3 := makeVersionId(t3)
|
||||
|
||||
entries := []*filer_pb.Entry{
|
||||
{Name: "v_" + vid1},
|
||||
{Name: "v_" + vid3},
|
||||
{Name: "v_" + vid2},
|
||||
}
|
||||
|
||||
sortVersionsByVersionId(entries)
|
||||
|
||||
// Should be sorted newest first: t3, t2, t1.
|
||||
expected := []string{"v_" + vid3, "v_" + vid2, "v_" + vid1}
|
||||
for i, want := range expected {
|
||||
if entries[i].Name != want {
|
||||
t.Errorf("entries[%d].Name = %s, want %s", i, entries[i].Name, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortVersionsByVersionId_SameTimestampDifferentSuffix(t *testing.T) {
|
||||
// Two versions with the same timestamp prefix but different random suffix.
|
||||
// The sort must still produce a deterministic order.
|
||||
base := makeVersionId(time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC))
|
||||
vid1 := base[:16] + "aaaaaaaaaaaaaaaa"
|
||||
vid2 := base[:16] + "bbbbbbbbbbbbbbbb"
|
||||
|
||||
entries := []*filer_pb.Entry{
|
||||
{Name: "v_" + vid2},
|
||||
{Name: "v_" + vid1},
|
||||
}
|
||||
|
||||
sortVersionsByVersionId(entries)
|
||||
|
||||
// New format: smaller hex = newer. vid1 ("aaa...") < vid2 ("bbb...") so vid1 is newer.
|
||||
if strings.TrimPrefix(entries[0].Name, "v_") != vid1 {
|
||||
t.Errorf("expected vid1 (newer) first, got %s", entries[0].Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareVersionIdsMixedFormats(t *testing.T) {
|
||||
// Old format: raw nanosecond timestamp (below threshold ~0x17...).
|
||||
// New format: inverted timestamp (above threshold ~0x68...).
|
||||
oldTs := time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC)
|
||||
newTs := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
oldFormatId := fmt.Sprintf("%016x", oldTs.UnixNano()) + "abcdef0123456789"
|
||||
newFormatId := makeVersionId(newTs) // uses inverted timestamp
|
||||
|
||||
// newTs is more recent, so newFormatId should sort as "newer".
|
||||
cmp := s3lifecycle.CompareVersionIds(newFormatId, oldFormatId)
|
||||
if cmp >= 0 {
|
||||
t.Errorf("expected new-format ID (2026) to be newer than old-format ID (2023), got cmp=%d", cmp)
|
||||
}
|
||||
|
||||
// Reverse comparison.
|
||||
cmp2 := s3lifecycle.CompareVersionIds(oldFormatId, newFormatId)
|
||||
if cmp2 <= 0 {
|
||||
t.Errorf("expected old-format ID (2023) to be older than new-format ID (2026), got cmp=%d", cmp2)
|
||||
}
|
||||
|
||||
// Sort a mixed slice: should be newest-first.
|
||||
entries := []*filer_pb.Entry{
|
||||
{Name: "v_" + oldFormatId},
|
||||
{Name: "v_" + newFormatId},
|
||||
}
|
||||
sortVersionsByVersionId(entries)
|
||||
|
||||
if strings.TrimPrefix(entries[0].Name, "v_") != newFormatId {
|
||||
t.Errorf("expected new-format (newer) entry first after sort")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionsDirectoryNaming(t *testing.T) {
|
||||
if s3_constants.VersionsFolder != ".versions" {
|
||||
t.Fatalf("unexpected VersionsFolder constant: %q", s3_constants.VersionsFolder)
|
||||
}
|
||||
|
||||
versionsDir := "/buckets/mybucket/path/to/key.versions"
|
||||
bucketPath := "/buckets/mybucket"
|
||||
relDir := strings.TrimPrefix(versionsDir, bucketPath+"/")
|
||||
objKey := strings.TrimSuffix(relDir, s3_constants.VersionsFolder)
|
||||
if objKey != "path/to/key" {
|
||||
t.Errorf("expected 'path/to/key', got %q", objKey)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user