s3lifecycle: add lifecycle rule evaluator package and extend XML types (#8807)
* s3api: extend lifecycle XML types with NoncurrentVersionExpiration, AbortIncompleteMultipartUpload Add missing S3 lifecycle rule types to the XML data model: - NoncurrentVersionExpiration with NoncurrentDays and NewerNoncurrentVersions - NoncurrentVersionTransition with NoncurrentDays and StorageClass - AbortIncompleteMultipartUpload with DaysAfterInitiation - Filter.ObjectSizeGreaterThan and ObjectSizeLessThan - And.ObjectSizeGreaterThan and ObjectSizeLessThan - Filter.UnmarshalXML to properly parse Tag, And, and size filter elements Each new type follows the existing set-field pattern for conditional XML marshaling. No behavior changes - these types are not yet wired into handlers or the lifecycle worker. * s3lifecycle: add lifecycle rule evaluator package New package weed/s3api/s3lifecycle/ provides a pure-function lifecycle rule evaluation engine. The evaluator accepts flattened Rule structs and ObjectInfo metadata, and returns the appropriate Action. Components: - evaluator.go: Evaluate() for per-object actions with S3 priority ordering (delete marker > noncurrent version > current expiration), ShouldExpireNoncurrentVersion() with NewerNoncurrentVersions support, EvaluateMPUAbort() for multipart upload rules - filter.go: prefix, tag, and size-based filter matching - tags.go: ExtractTags() extracts S3 tags from filer Extended metadata, HasTagRules() for scan-time optimization - version_time.go: GetVersionTimestamp() extracts timestamps from SeaweedFS version IDs (both old and new format) Comprehensive test coverage: 54 tests covering all action types, filter combinations, edge cases, and version ID formats. * s3api: add UnmarshalXML for Expiration, Transition, ExpireDeleteMarker Add UnmarshalXML methods that set the internal 'set' flag during XML parsing. Previously these flags were only set programmatically, causing XML round-trip to drop elements. This ensures lifecycle configurations stored as XML survive unmarshal/marshal cycles correctly. Add comprehensive XML round-trip tests for all lifecycle rule types including NoncurrentVersionExpiration, AbortIncompleteMultipartUpload, Filter with Tag/And/size constraints, and a complete Terraform-style lifecycle configuration. * s3lifecycle: address review feedback - Fix version_time.go overflow: guard timestampPart > MaxInt64 before the inversion subtraction to prevent uint64 wrap - Make all expiry checks inclusive (!now.Before instead of now.After) so actions trigger at the exact scheduled instant - Add NoncurrentIndex to ObjectInfo so Evaluate() can properly handle NewerNoncurrentVersions via ShouldExpireNoncurrentVersion() - Add test for high-bit overflow version ID * s3lifecycle: guard ShouldExpireNoncurrentVersion against zero SuccessorModTime Add early return when obj.IsLatest or obj.SuccessorModTime.IsZero() to prevent premature expiration of versions with uninitialized successor timestamps (zero value would compute to epoch, always expired). --------- Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
@@ -22,13 +22,16 @@ type Lifecycle struct {
|
|||||||
|
|
||||||
// Rule - a rule for lifecycle configuration.
|
// Rule - a rule for lifecycle configuration.
|
||||||
type Rule struct {
|
type Rule struct {
|
||||||
XMLName xml.Name `xml:"Rule"`
|
XMLName xml.Name `xml:"Rule"`
|
||||||
ID string `xml:"ID,omitempty"`
|
ID string `xml:"ID,omitempty"`
|
||||||
Status ruleStatus `xml:"Status"`
|
Status ruleStatus `xml:"Status"`
|
||||||
Filter Filter `xml:"Filter,omitempty"`
|
Filter Filter `xml:"Filter,omitempty"`
|
||||||
Prefix Prefix `xml:"Prefix,omitempty"`
|
Prefix Prefix `xml:"Prefix,omitempty"`
|
||||||
Expiration Expiration `xml:"Expiration,omitempty"`
|
Expiration Expiration `xml:"Expiration,omitempty"`
|
||||||
Transition Transition `xml:"Transition,omitempty"`
|
Transition Transition `xml:"Transition,omitempty"`
|
||||||
|
NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
||||||
|
NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
|
||||||
|
AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Filter - a filter for a lifecycle configuration Rule.
|
// Filter - a filter for a lifecycle configuration Rule.
|
||||||
@@ -43,6 +46,9 @@ type Filter struct {
|
|||||||
|
|
||||||
Tag Tag
|
Tag Tag
|
||||||
tagSet bool
|
tagSet bool
|
||||||
|
|
||||||
|
ObjectSizeGreaterThan int64
|
||||||
|
ObjectSizeLessThan int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefix holds the prefix xml tag in <Rule> and <Filter>
|
// Prefix holds the prefix xml tag in <Rule> and <Filter>
|
||||||
@@ -80,17 +86,83 @@ func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
|||||||
if err := e.EncodeToken(start); err != nil {
|
if err := e.EncodeToken(start); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil {
|
if f.andSet {
|
||||||
return err
|
if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if f.tagSet {
|
||||||
|
if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if f.ObjectSizeGreaterThan > 0 {
|
||||||
|
if err := e.EncodeElement(f.ObjectSizeGreaterThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeGreaterThan"}}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if f.ObjectSizeLessThan > 0 {
|
||||||
|
if err := e.EncodeElement(f.ObjectSizeLessThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeLessThan"}}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return e.EncodeToken(xml.EndElement{Name: start.Name})
|
return e.EncodeToken(xml.EndElement{Name: start.Name})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UnmarshalXML decodes Filter from XML, handling all child elements.
|
||||||
|
func (f *Filter) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
f.set = true
|
||||||
|
for {
|
||||||
|
tok, err := d.Token()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
switch t := tok.(type) {
|
||||||
|
case xml.StartElement:
|
||||||
|
switch t.Name.Local {
|
||||||
|
case "Prefix":
|
||||||
|
if err := d.DecodeElement(&f.Prefix, &t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case "Tag":
|
||||||
|
f.tagSet = true
|
||||||
|
if err := d.DecodeElement(&f.Tag, &t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case "And":
|
||||||
|
f.andSet = true
|
||||||
|
if err := d.DecodeElement(&f.And, &t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case "ObjectSizeGreaterThan":
|
||||||
|
if err := d.DecodeElement(&f.ObjectSizeGreaterThan, &t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case "ObjectSizeLessThan":
|
||||||
|
if err := d.DecodeElement(&f.ObjectSizeLessThan, &t); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if err := d.Skip(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case xml.EndElement:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// And - a tag to combine a prefix and multiple tags for lifecycle configuration rule.
|
// And - a tag to combine a prefix and multiple tags for lifecycle configuration rule.
|
||||||
type And struct {
|
type And struct {
|
||||||
XMLName xml.Name `xml:"And"`
|
XMLName xml.Name `xml:"And"`
|
||||||
Prefix Prefix `xml:"Prefix,omitempty"`
|
Prefix Prefix `xml:"Prefix,omitempty"`
|
||||||
Tags []Tag `xml:"Tag,omitempty"`
|
Tags []Tag `xml:"Tag,omitempty"`
|
||||||
|
ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"`
|
||||||
|
ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Expiration - expiration actions for a rule in lifecycle configuration.
|
// Expiration - expiration actions for a rule in lifecycle configuration.
|
||||||
@@ -112,6 +184,17 @@ func (e Expiration) MarshalXML(enc *xml.Encoder, startElement xml.StartElement)
|
|||||||
return enc.EncodeElement(expirationWrapper(e), startElement)
|
return enc.EncodeElement(expirationWrapper(e), startElement)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (e *Expiration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
type wrapper Expiration
|
||||||
|
var w wrapper
|
||||||
|
if err := d.DecodeElement(&w, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*e = Expiration(w)
|
||||||
|
e.set = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element.
|
// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element.
|
||||||
type ExpireDeleteMarker struct {
|
type ExpireDeleteMarker struct {
|
||||||
val bool
|
val bool
|
||||||
@@ -126,6 +209,15 @@ func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartEle
|
|||||||
return e.EncodeElement(b.val, startElement)
|
return e.EncodeElement(b.val, startElement)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (b *ExpireDeleteMarker) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
var v bool
|
||||||
|
if err := d.DecodeElement(&v, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*b = ExpireDeleteMarker{val: v, set: true}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ExpirationDate is a embedded type containing time.Time to unmarshal
|
// ExpirationDate is a embedded type containing time.Time to unmarshal
|
||||||
// Date in Expiration
|
// Date in Expiration
|
||||||
type ExpirationDate struct {
|
type ExpirationDate struct {
|
||||||
@@ -160,5 +252,102 @@ func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
|
|||||||
return enc.EncodeElement(transitionWrapper(t), start)
|
return enc.EncodeElement(transitionWrapper(t), start)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (t *Transition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
type wrapper Transition
|
||||||
|
var w wrapper
|
||||||
|
if err := d.DecodeElement(&w, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*t = Transition(w)
|
||||||
|
t.set = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// TransitionDays is a type alias to unmarshal Days in Transition
|
// TransitionDays is a type alias to unmarshal Days in Transition
|
||||||
type TransitionDays int
|
type TransitionDays int
|
||||||
|
|
||||||
|
// NoncurrentVersionExpiration - expiration actions for non-current object versions.
|
||||||
|
type NoncurrentVersionExpiration struct {
|
||||||
|
XMLName xml.Name `xml:"NoncurrentVersionExpiration"`
|
||||||
|
NoncurrentDays int `xml:"NoncurrentDays,omitempty"`
|
||||||
|
NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty"`
|
||||||
|
|
||||||
|
set bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalXML encodes NoncurrentVersionExpiration field into an XML form.
|
||||||
|
func (n NoncurrentVersionExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
|
||||||
|
if !n.set {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
type wrapper NoncurrentVersionExpiration
|
||||||
|
return enc.EncodeElement(wrapper(n), start)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
type wrapper NoncurrentVersionExpiration
|
||||||
|
var w wrapper
|
||||||
|
if err := d.DecodeElement(&w, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*n = NoncurrentVersionExpiration(w)
|
||||||
|
n.set = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NoncurrentVersionTransition - transition actions for non-current object versions.
|
||||||
|
type NoncurrentVersionTransition struct {
|
||||||
|
XMLName xml.Name `xml:"NoncurrentVersionTransition"`
|
||||||
|
NoncurrentDays int `xml:"NoncurrentDays,omitempty"`
|
||||||
|
StorageClass string `xml:"StorageClass,omitempty"`
|
||||||
|
|
||||||
|
set bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalXML encodes NoncurrentVersionTransition field into an XML form.
|
||||||
|
func (n NoncurrentVersionTransition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
|
||||||
|
if !n.set {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
type wrapper NoncurrentVersionTransition
|
||||||
|
return enc.EncodeElement(wrapper(n), start)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
type wrapper NoncurrentVersionTransition
|
||||||
|
var w wrapper
|
||||||
|
if err := d.DecodeElement(&w, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*n = NoncurrentVersionTransition(w)
|
||||||
|
n.set = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AbortIncompleteMultipartUpload - abort action for incomplete multipart uploads.
|
||||||
|
type AbortIncompleteMultipartUpload struct {
|
||||||
|
XMLName xml.Name `xml:"AbortIncompleteMultipartUpload"`
|
||||||
|
DaysAfterInitiation int `xml:"DaysAfterInitiation,omitempty"`
|
||||||
|
|
||||||
|
set bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalXML encodes AbortIncompleteMultipartUpload field into an XML form.
|
||||||
|
func (a AbortIncompleteMultipartUpload) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
|
||||||
|
if !a.set {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
type wrapper AbortIncompleteMultipartUpload
|
||||||
|
return enc.EncodeElement(wrapper(a), start)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *AbortIncompleteMultipartUpload) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
|
||||||
|
type wrapper AbortIncompleteMultipartUpload
|
||||||
|
var w wrapper
|
||||||
|
if err := d.DecodeElement(&w, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*a = AbortIncompleteMultipartUpload(w)
|
||||||
|
a.set = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
231
weed/s3api/s3api_policy_test.go
Normal file
231
weed/s3api/s3api_policy_test.go
Normal file
@@ -0,0 +1,231 @@
|
|||||||
|
package s3api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLifecycleXMLRoundTrip_NoncurrentVersionExpiration(t *testing.T) {
|
||||||
|
input := `<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<ID>expire-noncurrent</ID>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter><Prefix></Prefix></Filter>
|
||||||
|
<NoncurrentVersionExpiration>
|
||||||
|
<NoncurrentDays>30</NoncurrentDays>
|
||||||
|
<NewerNoncurrentVersions>2</NewerNoncurrentVersions>
|
||||||
|
</NoncurrentVersionExpiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>`
|
||||||
|
|
||||||
|
var lc Lifecycle
|
||||||
|
if err := xml.Unmarshal([]byte(input), &lc); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(lc.Rules) != 1 {
|
||||||
|
t.Fatalf("expected 1 rule, got %d", len(lc.Rules))
|
||||||
|
}
|
||||||
|
rule := lc.Rules[0]
|
||||||
|
if rule.ID != "expire-noncurrent" {
|
||||||
|
t.Errorf("expected ID 'expire-noncurrent', got %q", rule.ID)
|
||||||
|
}
|
||||||
|
if rule.NoncurrentVersionExpiration.NoncurrentDays != 30 {
|
||||||
|
t.Errorf("expected NoncurrentDays=30, got %d", rule.NoncurrentVersionExpiration.NoncurrentDays)
|
||||||
|
}
|
||||||
|
if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions != 2 {
|
||||||
|
t.Errorf("expected NewerNoncurrentVersions=2, got %d", rule.NoncurrentVersionExpiration.NewerNoncurrentVersions)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-marshal and verify it round-trips.
|
||||||
|
out, err := xml.Marshal(lc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
if !strings.Contains(s, "<NoncurrentDays>30</NoncurrentDays>") {
|
||||||
|
t.Errorf("marshaled XML missing NoncurrentDays: %s", s)
|
||||||
|
}
|
||||||
|
if !strings.Contains(s, "<NewerNoncurrentVersions>2</NewerNoncurrentVersions>") {
|
||||||
|
t.Errorf("marshaled XML missing NewerNoncurrentVersions: %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLifecycleXMLRoundTrip_AbortIncompleteMultipartUpload(t *testing.T) {
|
||||||
|
input := `<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<ID>abort-mpu</ID>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter><Prefix></Prefix></Filter>
|
||||||
|
<AbortIncompleteMultipartUpload>
|
||||||
|
<DaysAfterInitiation>7</DaysAfterInitiation>
|
||||||
|
</AbortIncompleteMultipartUpload>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>`
|
||||||
|
|
||||||
|
var lc Lifecycle
|
||||||
|
if err := xml.Unmarshal([]byte(input), &lc); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rule := lc.Rules[0]
|
||||||
|
if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != 7 {
|
||||||
|
t.Errorf("expected DaysAfterInitiation=7, got %d", rule.AbortIncompleteMultipartUpload.DaysAfterInitiation)
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := xml.Marshal(lc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(out), "<DaysAfterInitiation>7</DaysAfterInitiation>") {
|
||||||
|
t.Errorf("marshaled XML missing DaysAfterInitiation: %s", string(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLifecycleXMLRoundTrip_FilterWithTag(t *testing.T) {
|
||||||
|
input := `<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<ID>tag-filter</ID>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter>
|
||||||
|
<Tag><Key>env</Key><Value>dev</Value></Tag>
|
||||||
|
</Filter>
|
||||||
|
<Expiration><Days>7</Days></Expiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>`
|
||||||
|
|
||||||
|
var lc Lifecycle
|
||||||
|
if err := xml.Unmarshal([]byte(input), &lc); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rule := lc.Rules[0]
|
||||||
|
if !rule.Filter.tagSet {
|
||||||
|
t.Error("expected Filter.tagSet to be true")
|
||||||
|
}
|
||||||
|
if rule.Filter.Tag.Key != "env" || rule.Filter.Tag.Value != "dev" {
|
||||||
|
t.Errorf("expected Tag{env:dev}, got Tag{%s:%s}", rule.Filter.Tag.Key, rule.Filter.Tag.Value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLifecycleXMLRoundTrip_FilterWithAnd(t *testing.T) {
|
||||||
|
input := `<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<ID>and-filter</ID>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter>
|
||||||
|
<And>
|
||||||
|
<Prefix>logs/</Prefix>
|
||||||
|
<Tag><Key>env</Key><Value>dev</Value></Tag>
|
||||||
|
<Tag><Key>tier</Key><Value>hot</Value></Tag>
|
||||||
|
<ObjectSizeGreaterThan>1024</ObjectSizeGreaterThan>
|
||||||
|
<ObjectSizeLessThan>1048576</ObjectSizeLessThan>
|
||||||
|
</And>
|
||||||
|
</Filter>
|
||||||
|
<Expiration><Days>7</Days></Expiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>`
|
||||||
|
|
||||||
|
var lc Lifecycle
|
||||||
|
if err := xml.Unmarshal([]byte(input), &lc); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rule := lc.Rules[0]
|
||||||
|
if !rule.Filter.andSet {
|
||||||
|
t.Error("expected Filter.andSet to be true")
|
||||||
|
}
|
||||||
|
if rule.Filter.And.Prefix.String() != "logs/" {
|
||||||
|
t.Errorf("expected And.Prefix='logs/', got %q", rule.Filter.And.Prefix.String())
|
||||||
|
}
|
||||||
|
if len(rule.Filter.And.Tags) != 2 {
|
||||||
|
t.Fatalf("expected 2 And tags, got %d", len(rule.Filter.And.Tags))
|
||||||
|
}
|
||||||
|
if rule.Filter.And.ObjectSizeGreaterThan != 1024 {
|
||||||
|
t.Errorf("expected ObjectSizeGreaterThan=1024, got %d", rule.Filter.And.ObjectSizeGreaterThan)
|
||||||
|
}
|
||||||
|
if rule.Filter.And.ObjectSizeLessThan != 1048576 {
|
||||||
|
t.Errorf("expected ObjectSizeLessThan=1048576, got %d", rule.Filter.And.ObjectSizeLessThan)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLifecycleXMLRoundTrip_FilterWithSizeOnly(t *testing.T) {
|
||||||
|
input := `<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<ID>size-filter</ID>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter>
|
||||||
|
<ObjectSizeGreaterThan>512</ObjectSizeGreaterThan>
|
||||||
|
<ObjectSizeLessThan>10485760</ObjectSizeLessThan>
|
||||||
|
</Filter>
|
||||||
|
<Expiration><Days>30</Days></Expiration>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>`
|
||||||
|
|
||||||
|
var lc Lifecycle
|
||||||
|
if err := xml.Unmarshal([]byte(input), &lc); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rule := lc.Rules[0]
|
||||||
|
if rule.Filter.ObjectSizeGreaterThan != 512 {
|
||||||
|
t.Errorf("expected ObjectSizeGreaterThan=512, got %d", rule.Filter.ObjectSizeGreaterThan)
|
||||||
|
}
|
||||||
|
if rule.Filter.ObjectSizeLessThan != 10485760 {
|
||||||
|
t.Errorf("expected ObjectSizeLessThan=10485760, got %d", rule.Filter.ObjectSizeLessThan)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLifecycleXMLRoundTrip_CompleteRule(t *testing.T) {
|
||||||
|
// A complete lifecycle config similar to what Terraform sends.
|
||||||
|
input := `<LifecycleConfiguration>
|
||||||
|
<Rule>
|
||||||
|
<ID>rotation</ID>
|
||||||
|
<Filter><Prefix></Prefix></Filter>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Expiration><Days>30</Days></Expiration>
|
||||||
|
<NoncurrentVersionExpiration>
|
||||||
|
<NoncurrentDays>1</NoncurrentDays>
|
||||||
|
</NoncurrentVersionExpiration>
|
||||||
|
<AbortIncompleteMultipartUpload>
|
||||||
|
<DaysAfterInitiation>1</DaysAfterInitiation>
|
||||||
|
</AbortIncompleteMultipartUpload>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>`
|
||||||
|
|
||||||
|
var lc Lifecycle
|
||||||
|
if err := xml.Unmarshal([]byte(input), &lc); err != nil {
|
||||||
|
t.Fatalf("unmarshal: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rule := lc.Rules[0]
|
||||||
|
if rule.ID != "rotation" {
|
||||||
|
t.Errorf("expected ID 'rotation', got %q", rule.ID)
|
||||||
|
}
|
||||||
|
if rule.Expiration.Days != 30 {
|
||||||
|
t.Errorf("expected Expiration.Days=30, got %d", rule.Expiration.Days)
|
||||||
|
}
|
||||||
|
if rule.NoncurrentVersionExpiration.NoncurrentDays != 1 {
|
||||||
|
t.Errorf("expected NoncurrentDays=1, got %d", rule.NoncurrentVersionExpiration.NoncurrentDays)
|
||||||
|
}
|
||||||
|
if rule.AbortIncompleteMultipartUpload.DaysAfterInitiation != 1 {
|
||||||
|
t.Errorf("expected DaysAfterInitiation=1, got %d", rule.AbortIncompleteMultipartUpload.DaysAfterInitiation)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-marshal and verify all fields survive.
|
||||||
|
out, err := xml.Marshal(lc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
s := string(out)
|
||||||
|
for _, expected := range []string{
|
||||||
|
"<Days>30</Days>",
|
||||||
|
"<NoncurrentDays>1</NoncurrentDays>",
|
||||||
|
"<DaysAfterInitiation>1</DaysAfterInitiation>",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(s, expected) {
|
||||||
|
t.Errorf("marshaled XML missing %q: %s", expected, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
127
weed/s3api/s3lifecycle/evaluator.go
Normal file
127
weed/s3api/s3lifecycle/evaluator.go
Normal file
@@ -0,0 +1,127 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// Evaluate checks the given lifecycle rules against an object and returns
|
||||||
|
// the highest-priority action that applies. The evaluation follows S3's
|
||||||
|
// action priority:
|
||||||
|
// 1. ExpiredObjectDeleteMarker (delete marker is sole version)
|
||||||
|
// 2. NoncurrentVersionExpiration (non-current version age/count)
|
||||||
|
// 3. Current version Expiration (Days or Date)
|
||||||
|
//
|
||||||
|
// AbortIncompleteMultipartUpload is evaluated separately since it applies
|
||||||
|
// to uploads, not objects. Use EvaluateMPUAbort for that.
|
||||||
|
func Evaluate(rules []Rule, obj ObjectInfo, now time.Time) EvalResult {
|
||||||
|
// Phase 1: ExpiredObjectDeleteMarker
|
||||||
|
if obj.IsDeleteMarker && obj.IsLatest && obj.NumVersions == 1 {
|
||||||
|
for _, rule := range rules {
|
||||||
|
if rule.Status != "Enabled" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !matchesFilter(rule, obj) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if rule.ExpiredObjectDeleteMarker {
|
||||||
|
return EvalResult{Action: ActionExpireDeleteMarker, RuleID: rule.ID}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 2: NoncurrentVersionExpiration
|
||||||
|
if !obj.IsLatest && !obj.SuccessorModTime.IsZero() {
|
||||||
|
for _, rule := range rules {
|
||||||
|
if ShouldExpireNoncurrentVersion(rule, obj, obj.NoncurrentIndex, now) {
|
||||||
|
return EvalResult{Action: ActionDeleteVersion, RuleID: rule.ID}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 3: Current version Expiration
|
||||||
|
if obj.IsLatest && !obj.IsDeleteMarker {
|
||||||
|
for _, rule := range rules {
|
||||||
|
if rule.Status != "Enabled" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !matchesFilter(rule, obj) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Date-based expiration
|
||||||
|
if !rule.ExpirationDate.IsZero() && !now.Before(rule.ExpirationDate) {
|
||||||
|
return EvalResult{Action: ActionDeleteObject, RuleID: rule.ID}
|
||||||
|
}
|
||||||
|
// Days-based expiration
|
||||||
|
if rule.ExpirationDays > 0 {
|
||||||
|
expiryTime := expectedExpiryTime(obj.ModTime, rule.ExpirationDays)
|
||||||
|
if !now.Before(expiryTime) {
|
||||||
|
return EvalResult{Action: ActionDeleteObject, RuleID: rule.ID}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return EvalResult{Action: ActionNone}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldExpireNoncurrentVersion checks whether a non-current version should
|
||||||
|
// be expired considering both NoncurrentDays and NewerNoncurrentVersions.
|
||||||
|
// noncurrentIndex is the 0-based position among non-current versions sorted
|
||||||
|
// newest-first (0 = newest non-current version).
|
||||||
|
func ShouldExpireNoncurrentVersion(rule Rule, obj ObjectInfo, noncurrentIndex int, now time.Time) bool {
|
||||||
|
if rule.Status != "Enabled" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if rule.NoncurrentVersionExpirationDays <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if obj.IsLatest || obj.SuccessorModTime.IsZero() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !matchesFilter(rule, obj) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check age threshold.
|
||||||
|
expiryTime := expectedExpiryTime(obj.SuccessorModTime, rule.NoncurrentVersionExpirationDays)
|
||||||
|
if now.Before(expiryTime) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check NewerNoncurrentVersions count threshold.
|
||||||
|
if rule.NewerNoncurrentVersions > 0 && noncurrentIndex < rule.NewerNoncurrentVersions {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// EvaluateMPUAbort finds the applicable AbortIncompleteMultipartUpload rule
|
||||||
|
// for a multipart upload with the given key prefix and creation time.
|
||||||
|
func EvaluateMPUAbort(rules []Rule, uploadKey string, createdAt time.Time, now time.Time) EvalResult {
|
||||||
|
for _, rule := range rules {
|
||||||
|
if rule.Status != "Enabled" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if rule.AbortMPUDaysAfterInitiation <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !matchesPrefix(rule.Prefix, uploadKey) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cutoff := createdAt.Add(time.Duration(rule.AbortMPUDaysAfterInitiation) * 24 * time.Hour)
|
||||||
|
if !now.Before(cutoff) {
|
||||||
|
return EvalResult{Action: ActionAbortMultipartUpload, RuleID: rule.ID}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return EvalResult{Action: ActionNone}
|
||||||
|
}
|
||||||
|
|
||||||
|
// expectedExpiryTime computes the expiration time given a reference time and
|
||||||
|
// a number of days. Following S3 semantics, expiration happens at midnight UTC
|
||||||
|
// of the day after the specified number of days.
|
||||||
|
func expectedExpiryTime(refTime time.Time, days int) time.Time {
|
||||||
|
if days == 0 {
|
||||||
|
return refTime
|
||||||
|
}
|
||||||
|
t := refTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour)
|
||||||
|
return t.Truncate(24 * time.Hour)
|
||||||
|
}
|
||||||
495
weed/s3api/s3lifecycle/evaluator_test.go
Normal file
495
weed/s3api/s3lifecycle/evaluator_test.go
Normal file
@@ -0,0 +1,495 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var now = time.Date(2026, 3, 27, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
func TestEvaluate_ExpirationDays(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "expire-30d", Status: "Enabled",
|
||||||
|
ExpirationDays: 30,
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("object_older_than_days_is_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "data/file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-31 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
assertEqual(t, "expire-30d", result.RuleID)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("object_younger_than_days_is_not_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "data/file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non_latest_version_not_affected_by_expiration_days", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "data/file.txt", IsLatest: false,
|
||||||
|
ModTime: now.Add(-60 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("delete_marker_not_affected_by_expiration_days", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "data/file.txt", IsLatest: true, IsDeleteMarker: true,
|
||||||
|
ModTime: now.Add(-60 * 24 * time.Hour), NumVersions: 3,
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_ExpirationDate(t *testing.T) {
|
||||||
|
expirationDate := time.Date(2026, 3, 15, 0, 0, 0, 0, time.UTC)
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "expire-date", Status: "Enabled",
|
||||||
|
ExpirationDate: expirationDate,
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("object_expired_after_date", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-60 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("object_not_expired_before_date", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-1 * time.Hour),
|
||||||
|
}
|
||||||
|
beforeDate := time.Date(2026, 3, 10, 0, 0, 0, 0, time.UTC)
|
||||||
|
result := Evaluate(rules, obj, beforeDate)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_ExpiredObjectDeleteMarker(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "cleanup-markers", Status: "Enabled",
|
||||||
|
ExpiredObjectDeleteMarker: true,
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("sole_delete_marker_is_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true, IsDeleteMarker: true,
|
||||||
|
NumVersions: 1,
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionExpireDeleteMarker, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("delete_marker_with_other_versions_not_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true, IsDeleteMarker: true,
|
||||||
|
NumVersions: 3,
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non_latest_delete_marker_not_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: false, IsDeleteMarker: true,
|
||||||
|
NumVersions: 1,
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non_delete_marker_not_affected", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true, IsDeleteMarker: false,
|
||||||
|
NumVersions: 1,
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_NoncurrentVersionExpiration(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "expire-noncurrent", Status: "Enabled",
|
||||||
|
NoncurrentVersionExpirationDays: 30,
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("old_noncurrent_version_is_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: false,
|
||||||
|
SuccessorModTime: now.Add(-45 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteVersion, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("recent_noncurrent_version_is_not_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: false,
|
||||||
|
SuccessorModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("latest_version_not_affected", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-60 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestShouldExpireNoncurrentVersion(t *testing.T) {
|
||||||
|
rule := Rule{
|
||||||
|
ID: "noncurrent-rule", Status: "Enabled",
|
||||||
|
NoncurrentVersionExpirationDays: 30,
|
||||||
|
NewerNoncurrentVersions: 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("old_version_beyond_count_is_expired", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: false,
|
||||||
|
SuccessorModTime: now.Add(-45 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
// noncurrentIndex=2 means this is the 3rd noncurrent version (0-indexed)
|
||||||
|
// With NewerNoncurrentVersions=2, indices 0 and 1 are kept.
|
||||||
|
if !ShouldExpireNoncurrentVersion(rule, obj, 2, now) {
|
||||||
|
t.Error("expected version at index 2 to be expired")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("old_version_within_count_is_kept", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: false,
|
||||||
|
SuccessorModTime: now.Add(-45 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
// noncurrentIndex=1 is within the keep threshold (NewerNoncurrentVersions=2).
|
||||||
|
if ShouldExpireNoncurrentVersion(rule, obj, 1, now) {
|
||||||
|
t.Error("expected version at index 1 to be kept")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("recent_version_beyond_count_is_kept", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: false,
|
||||||
|
SuccessorModTime: now.Add(-5 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
// Even at index 5 (beyond count), if too young, it's kept.
|
||||||
|
if ShouldExpireNoncurrentVersion(rule, obj, 5, now) {
|
||||||
|
t.Error("expected recent version to be kept regardless of index")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("disabled_rule_never_expires", func(t *testing.T) {
|
||||||
|
disabled := Rule{
|
||||||
|
ID: "disabled", Status: "Disabled",
|
||||||
|
NoncurrentVersionExpirationDays: 1,
|
||||||
|
}
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: false,
|
||||||
|
SuccessorModTime: now.Add(-365 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
if ShouldExpireNoncurrentVersion(disabled, obj, 10, now) {
|
||||||
|
t.Error("disabled rule should never expire")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_PrefixFilter(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "logs-only", Status: "Enabled",
|
||||||
|
Prefix: "logs/",
|
||||||
|
ExpirationDays: 7,
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("matching_prefix", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "logs/app.log", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("non_matching_prefix", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "data/file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_TagFilter(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "temp-only", Status: "Enabled",
|
||||||
|
ExpirationDays: 1,
|
||||||
|
FilterTags: map[string]string{"env": "temp"},
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("matching_tags", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-5 * 24 * time.Hour),
|
||||||
|
Tags: map[string]string{"env": "temp", "project": "foo"},
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("missing_tag", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-5 * 24 * time.Hour),
|
||||||
|
Tags: map[string]string{"project": "foo"},
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("wrong_tag_value", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-5 * 24 * time.Hour),
|
||||||
|
Tags: map[string]string{"env": "prod"},
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nil_object_tags", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-5 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_SizeFilter(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "large-files", Status: "Enabled",
|
||||||
|
ExpirationDays: 7,
|
||||||
|
FilterSizeGreaterThan: 1024 * 1024, // > 1 MB
|
||||||
|
FilterSizeLessThan: 100 * 1024 * 1024, // < 100 MB
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("matching_size", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.bin", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
Size: 10 * 1024 * 1024, // 10 MB
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("too_small", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.bin", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
Size: 512, // 512 bytes
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("too_large", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.bin", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
Size: 200 * 1024 * 1024, // 200 MB
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_CombinedFilters(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "combined", Status: "Enabled",
|
||||||
|
Prefix: "logs/",
|
||||||
|
ExpirationDays: 7,
|
||||||
|
FilterTags: map[string]string{"env": "dev"},
|
||||||
|
FilterSizeGreaterThan: 100,
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("all_filters_match", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "logs/app.log", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
Size: 1024,
|
||||||
|
Tags: map[string]string{"env": "dev"},
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("prefix_doesnt_match", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "data/app.log", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
Size: 1024,
|
||||||
|
Tags: map[string]string{"env": "dev"},
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("tag_doesnt_match", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "logs/app.log", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
Size: 1024,
|
||||||
|
Tags: map[string]string{"env": "prod"},
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("size_doesnt_match", func(t *testing.T) {
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "logs/app.log", IsLatest: true,
|
||||||
|
ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
Size: 50, // too small
|
||||||
|
Tags: map[string]string{"env": "dev"},
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_DisabledRule(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "disabled", Status: "Disabled",
|
||||||
|
ExpirationDays: 1,
|
||||||
|
}}
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-365 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_MultipleRules_Priority(t *testing.T) {
|
||||||
|
t.Run("delete_marker_takes_priority_over_expiration", func(t *testing.T) {
|
||||||
|
rules := []Rule{
|
||||||
|
{ID: "expire", Status: "Enabled", ExpirationDays: 1},
|
||||||
|
{ID: "marker", Status: "Enabled", ExpiredObjectDeleteMarker: true},
|
||||||
|
}
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "file.txt", IsLatest: true, IsDeleteMarker: true,
|
||||||
|
NumVersions: 1, ModTime: now.Add(-10 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionExpireDeleteMarker, result.Action)
|
||||||
|
assertEqual(t, "marker", result.RuleID)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("first_matching_expiration_rule_wins", func(t *testing.T) {
|
||||||
|
rules := []Rule{
|
||||||
|
{ID: "rule1", Status: "Enabled", ExpirationDays: 30, Prefix: "logs/"},
|
||||||
|
{ID: "rule2", Status: "Enabled", ExpirationDays: 7},
|
||||||
|
}
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "logs/app.log", IsLatest: true,
|
||||||
|
ModTime: now.Add(-31 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
assertEqual(t, "rule1", result.RuleID)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluate_EmptyPrefix(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "all", Status: "Enabled",
|
||||||
|
ExpirationDays: 30,
|
||||||
|
}}
|
||||||
|
obj := ObjectInfo{
|
||||||
|
Key: "any/path/file.txt", IsLatest: true,
|
||||||
|
ModTime: now.Add(-31 * 24 * time.Hour),
|
||||||
|
}
|
||||||
|
result := Evaluate(rules, obj, now)
|
||||||
|
assertAction(t, ActionDeleteObject, result.Action)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEvaluateMPUAbort(t *testing.T) {
|
||||||
|
rules := []Rule{{
|
||||||
|
ID: "abort-mpu", Status: "Enabled",
|
||||||
|
AbortMPUDaysAfterInitiation: 7,
|
||||||
|
}}
|
||||||
|
|
||||||
|
t.Run("old_upload_is_aborted", func(t *testing.T) {
|
||||||
|
result := EvaluateMPUAbort(rules, "uploads/file.bin", now.Add(-10*24*time.Hour), now)
|
||||||
|
assertAction(t, ActionAbortMultipartUpload, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("recent_upload_is_not_aborted", func(t *testing.T) {
|
||||||
|
result := EvaluateMPUAbort(rules, "uploads/file.bin", now.Add(-3*24*time.Hour), now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("prefix_scoped_abort", func(t *testing.T) {
|
||||||
|
prefixRules := []Rule{{
|
||||||
|
ID: "abort-logs", Status: "Enabled",
|
||||||
|
Prefix: "logs/",
|
||||||
|
AbortMPUDaysAfterInitiation: 1,
|
||||||
|
}}
|
||||||
|
result := EvaluateMPUAbort(prefixRules, "data/file.bin", now.Add(-5*24*time.Hour), now)
|
||||||
|
assertAction(t, ActionNone, result.Action)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExpectedExpiryTime(t *testing.T) {
|
||||||
|
ref := time.Date(2026, 3, 1, 15, 30, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
t.Run("30_days", func(t *testing.T) {
|
||||||
|
// S3 spec: expires at midnight UTC of day 32 (ref + 31 days, truncated).
|
||||||
|
expiry := expectedExpiryTime(ref, 30)
|
||||||
|
expected := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
|
||||||
|
if !expiry.Equal(expected) {
|
||||||
|
t.Errorf("expected %v, got %v", expected, expiry)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("zero_days_returns_ref", func(t *testing.T) {
|
||||||
|
expiry := expectedExpiryTime(ref, 0)
|
||||||
|
if !expiry.Equal(ref) {
|
||||||
|
t.Errorf("expected %v, got %v", ref, expiry)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertAction(t *testing.T, expected, actual Action) {
|
||||||
|
t.Helper()
|
||||||
|
if expected != actual {
|
||||||
|
t.Errorf("expected action %d, got %d", expected, actual)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertEqual(t *testing.T, expected, actual string) {
|
||||||
|
t.Helper()
|
||||||
|
if expected != actual {
|
||||||
|
t.Errorf("expected %q, got %q", expected, actual)
|
||||||
|
}
|
||||||
|
}
|
||||||
56
weed/s3api/s3lifecycle/filter.go
Normal file
56
weed/s3api/s3lifecycle/filter.go
Normal file
@@ -0,0 +1,56 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
// matchesFilter checks if an object matches the rule's filter criteria
|
||||||
|
// (prefix, tags, and size constraints).
|
||||||
|
func matchesFilter(rule Rule, obj ObjectInfo) bool {
|
||||||
|
if !matchesPrefix(rule.Prefix, obj.Key) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !matchesTags(rule.FilterTags, obj.Tags) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !matchesSize(rule.FilterSizeGreaterThan, rule.FilterSizeLessThan, obj.Size) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchesPrefix returns true if the object key starts with the given prefix.
|
||||||
|
// An empty prefix matches all keys.
|
||||||
|
func matchesPrefix(prefix, key string) bool {
|
||||||
|
if prefix == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return strings.HasPrefix(key, prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchesTags returns true if all rule tags are present in the object's tags
|
||||||
|
// with matching values. An empty or nil rule tag set matches all objects.
|
||||||
|
func matchesTags(ruleTags, objTags map[string]string) bool {
|
||||||
|
if len(ruleTags) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(objTags) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for k, v := range ruleTags {
|
||||||
|
if objVal, ok := objTags[k]; !ok || objVal != v {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchesSize returns true if the object's size falls within the specified
|
||||||
|
// bounds. Zero values mean no constraint on that side.
|
||||||
|
func matchesSize(greaterThan, lessThan, objSize int64) bool {
|
||||||
|
if greaterThan > 0 && objSize <= greaterThan {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if lessThan > 0 && objSize >= lessThan {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
79
weed/s3api/s3lifecycle/filter_test.go
Normal file
79
weed/s3api/s3lifecycle/filter_test.go
Normal file
@@ -0,0 +1,79 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestMatchesPrefix(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
prefix string
|
||||||
|
key string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"empty_prefix_matches_all", "", "any/key.txt", true},
|
||||||
|
{"exact_prefix_match", "logs/", "logs/app.log", true},
|
||||||
|
{"prefix_mismatch", "logs/", "data/file.txt", false},
|
||||||
|
{"key_shorter_than_prefix", "very/long/prefix/", "short", false},
|
||||||
|
{"prefix_equals_key", "exact", "exact", true},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := matchesPrefix(tt.prefix, tt.key); got != tt.want {
|
||||||
|
t.Errorf("matchesPrefix(%q, %q) = %v, want %v", tt.prefix, tt.key, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMatchesTags(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
ruleTags map[string]string
|
||||||
|
objTags map[string]string
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"nil_rule_tags_match_all", nil, map[string]string{"a": "1"}, true},
|
||||||
|
{"empty_rule_tags_match_all", map[string]string{}, map[string]string{"a": "1"}, true},
|
||||||
|
{"nil_obj_tags_no_match", map[string]string{"a": "1"}, nil, false},
|
||||||
|
{"single_tag_match", map[string]string{"env": "dev"}, map[string]string{"env": "dev", "foo": "bar"}, true},
|
||||||
|
{"single_tag_value_mismatch", map[string]string{"env": "dev"}, map[string]string{"env": "prod"}, false},
|
||||||
|
{"single_tag_key_missing", map[string]string{"env": "dev"}, map[string]string{"foo": "bar"}, false},
|
||||||
|
{"multi_tag_all_match", map[string]string{"env": "dev", "tier": "hot"}, map[string]string{"env": "dev", "tier": "hot", "extra": "x"}, true},
|
||||||
|
{"multi_tag_partial_match", map[string]string{"env": "dev", "tier": "hot"}, map[string]string{"env": "dev"}, false},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := matchesTags(tt.ruleTags, tt.objTags); got != tt.want {
|
||||||
|
t.Errorf("matchesTags() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMatchesSize(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
greaterThan int64
|
||||||
|
lessThan int64
|
||||||
|
objSize int64
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"no_constraints", 0, 0, 1000, true},
|
||||||
|
{"only_greater_than_pass", 100, 0, 200, true},
|
||||||
|
{"only_greater_than_fail", 100, 0, 50, false},
|
||||||
|
{"only_greater_than_equal_fail", 100, 0, 100, false},
|
||||||
|
{"only_less_than_pass", 0, 1000, 500, true},
|
||||||
|
{"only_less_than_fail", 0, 1000, 2000, false},
|
||||||
|
{"only_less_than_equal_fail", 0, 1000, 1000, false},
|
||||||
|
{"both_constraints_pass", 100, 1000, 500, true},
|
||||||
|
{"both_constraints_too_small", 100, 1000, 50, false},
|
||||||
|
{"both_constraints_too_large", 100, 1000, 2000, false},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := matchesSize(tt.greaterThan, tt.lessThan, tt.objSize); got != tt.want {
|
||||||
|
t.Errorf("matchesSize(%d, %d, %d) = %v, want %v",
|
||||||
|
tt.greaterThan, tt.lessThan, tt.objSize, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
95
weed/s3api/s3lifecycle/rule.go
Normal file
95
weed/s3api/s3lifecycle/rule.go
Normal file
@@ -0,0 +1,95 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// Rule is a flattened, evaluator-friendly representation of an S3 lifecycle rule.
|
||||||
|
// Callers convert from the XML-parsed s3api.Rule (which has nested structs with
|
||||||
|
// set-flags for conditional XML marshaling) to this type.
|
||||||
|
type Rule struct {
|
||||||
|
ID string
|
||||||
|
Status string // "Enabled" or "Disabled"
|
||||||
|
|
||||||
|
// Prefix filter (from Rule.Prefix or Rule.Filter.Prefix or Rule.Filter.And.Prefix).
|
||||||
|
Prefix string
|
||||||
|
|
||||||
|
// Expiration for current versions.
|
||||||
|
ExpirationDays int
|
||||||
|
ExpirationDate time.Time
|
||||||
|
ExpiredObjectDeleteMarker bool
|
||||||
|
|
||||||
|
// Expiration for non-current versions.
|
||||||
|
NoncurrentVersionExpirationDays int
|
||||||
|
NewerNoncurrentVersions int
|
||||||
|
|
||||||
|
// Abort incomplete multipart uploads.
|
||||||
|
AbortMPUDaysAfterInitiation int
|
||||||
|
|
||||||
|
// Tag filter (from Rule.Filter.Tag or Rule.Filter.And.Tags).
|
||||||
|
FilterTags map[string]string
|
||||||
|
|
||||||
|
// Size filters.
|
||||||
|
FilterSizeGreaterThan int64
|
||||||
|
FilterSizeLessThan int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObjectInfo is the metadata about an object that the evaluator uses to
|
||||||
|
// determine which lifecycle action applies. Callers build this from filer
|
||||||
|
// entry attributes and extended metadata.
|
||||||
|
type ObjectInfo struct {
|
||||||
|
// Key is the object key relative to the bucket root.
|
||||||
|
Key string
|
||||||
|
|
||||||
|
// ModTime is the object's modification time (entry.Attributes.Mtime).
|
||||||
|
ModTime time.Time
|
||||||
|
|
||||||
|
// Size is the object size in bytes (entry.Attributes.FileSize).
|
||||||
|
Size int64
|
||||||
|
|
||||||
|
// IsLatest is true if this is the current version of the object.
|
||||||
|
IsLatest bool
|
||||||
|
|
||||||
|
// IsDeleteMarker is true if this entry is an S3 delete marker.
|
||||||
|
IsDeleteMarker bool
|
||||||
|
|
||||||
|
// NumVersions is the total number of versions for this object key,
|
||||||
|
// including delete markers. Used for ExpiredObjectDeleteMarker evaluation.
|
||||||
|
NumVersions int
|
||||||
|
|
||||||
|
// SuccessorModTime is the creation time of the version that replaced
|
||||||
|
// this one (making it non-current). Derived from the successor's version
|
||||||
|
// ID timestamp. Zero value for the latest version.
|
||||||
|
SuccessorModTime time.Time
|
||||||
|
|
||||||
|
// NoncurrentIndex is the 0-based position among non-current versions
|
||||||
|
// sorted newest-first (0 = newest non-current version). Used by
|
||||||
|
// NewerNoncurrentVersions evaluation. -1 or unset for current versions.
|
||||||
|
NoncurrentIndex int
|
||||||
|
|
||||||
|
// Tags are the object's user-defined tags, extracted from the entry's
|
||||||
|
// Extended metadata (keys prefixed with "X-Amz-Tagging-").
|
||||||
|
Tags map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Action represents the lifecycle action to take on an object.
|
||||||
|
type Action int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ActionNone means no lifecycle rule applies.
|
||||||
|
ActionNone Action = iota
|
||||||
|
// ActionDeleteObject deletes the current version of the object.
|
||||||
|
ActionDeleteObject
|
||||||
|
// ActionDeleteVersion deletes a specific non-current version.
|
||||||
|
ActionDeleteVersion
|
||||||
|
// ActionExpireDeleteMarker removes a delete marker that is the sole remaining version.
|
||||||
|
ActionExpireDeleteMarker
|
||||||
|
// ActionAbortMultipartUpload aborts an incomplete multipart upload.
|
||||||
|
ActionAbortMultipartUpload
|
||||||
|
)
|
||||||
|
|
||||||
|
// EvalResult is the output of lifecycle rule evaluation.
|
||||||
|
type EvalResult struct {
|
||||||
|
// Action is the lifecycle action to take.
|
||||||
|
Action Action
|
||||||
|
// RuleID is the ID of the rule that triggered this action.
|
||||||
|
RuleID string
|
||||||
|
}
|
||||||
34
weed/s3api/s3lifecycle/tags.go
Normal file
34
weed/s3api/s3lifecycle/tags.go
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import "strings"
|
||||||
|
|
||||||
|
const tagPrefix = "X-Amz-Tagging-"
|
||||||
|
|
||||||
|
// ExtractTags extracts S3 object tags from a filer entry's Extended metadata.
|
||||||
|
// Tags are stored with the key prefix "X-Amz-Tagging-" followed by the tag key.
|
||||||
|
func ExtractTags(extended map[string][]byte) map[string]string {
|
||||||
|
if len(extended) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var tags map[string]string
|
||||||
|
for k, v := range extended {
|
||||||
|
if strings.HasPrefix(k, tagPrefix) {
|
||||||
|
if tags == nil {
|
||||||
|
tags = make(map[string]string)
|
||||||
|
}
|
||||||
|
tags[k[len(tagPrefix):]] = string(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tags
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasTagRules returns true if any enabled rule in the set uses tag-based filtering.
|
||||||
|
// This is used as an optimization to skip tag extraction when no rules need it.
|
||||||
|
func HasTagRules(rules []Rule) bool {
|
||||||
|
for _, r := range rules {
|
||||||
|
if r.Status == "Enabled" && len(r.FilterTags) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
89
weed/s3api/s3lifecycle/tags_test.go
Normal file
89
weed/s3api/s3lifecycle/tags_test.go
Normal file
@@ -0,0 +1,89 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestExtractTags(t *testing.T) {
|
||||||
|
t.Run("extracts_tags_with_prefix", func(t *testing.T) {
|
||||||
|
extended := map[string][]byte{
|
||||||
|
"X-Amz-Tagging-env": []byte("prod"),
|
||||||
|
"X-Amz-Tagging-project": []byte("foo"),
|
||||||
|
"Content-Type": []byte("text/plain"),
|
||||||
|
"X-Amz-Meta-Custom": []byte("value"),
|
||||||
|
}
|
||||||
|
tags := ExtractTags(extended)
|
||||||
|
if len(tags) != 2 {
|
||||||
|
t.Fatalf("expected 2 tags, got %d", len(tags))
|
||||||
|
}
|
||||||
|
if tags["env"] != "prod" {
|
||||||
|
t.Errorf("expected env=prod, got %q", tags["env"])
|
||||||
|
}
|
||||||
|
if tags["project"] != "foo" {
|
||||||
|
t.Errorf("expected project=foo, got %q", tags["project"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nil_extended_returns_nil", func(t *testing.T) {
|
||||||
|
tags := ExtractTags(nil)
|
||||||
|
if tags != nil {
|
||||||
|
t.Errorf("expected nil, got %v", tags)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no_tags_returns_nil", func(t *testing.T) {
|
||||||
|
extended := map[string][]byte{
|
||||||
|
"Content-Type": []byte("text/plain"),
|
||||||
|
}
|
||||||
|
tags := ExtractTags(extended)
|
||||||
|
if tags != nil {
|
||||||
|
t.Errorf("expected nil, got %v", tags)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty_tag_value", func(t *testing.T) {
|
||||||
|
extended := map[string][]byte{
|
||||||
|
"X-Amz-Tagging-empty": []byte(""),
|
||||||
|
}
|
||||||
|
tags := ExtractTags(extended)
|
||||||
|
if len(tags) != 1 {
|
||||||
|
t.Fatalf("expected 1 tag, got %d", len(tags))
|
||||||
|
}
|
||||||
|
if tags["empty"] != "" {
|
||||||
|
t.Errorf("expected empty value, got %q", tags["empty"])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHasTagRules(t *testing.T) {
|
||||||
|
t.Run("has_tag_rules", func(t *testing.T) {
|
||||||
|
rules := []Rule{
|
||||||
|
{Status: "Enabled", FilterTags: map[string]string{"env": "dev"}},
|
||||||
|
}
|
||||||
|
if !HasTagRules(rules) {
|
||||||
|
t.Error("expected true")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("no_tag_rules", func(t *testing.T) {
|
||||||
|
rules := []Rule{
|
||||||
|
{Status: "Enabled", ExpirationDays: 30},
|
||||||
|
}
|
||||||
|
if HasTagRules(rules) {
|
||||||
|
t.Error("expected false")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("disabled_tag_rule", func(t *testing.T) {
|
||||||
|
rules := []Rule{
|
||||||
|
{Status: "Disabled", FilterTags: map[string]string{"env": "dev"}},
|
||||||
|
}
|
||||||
|
if HasTagRules(rules) {
|
||||||
|
t.Error("expected false for disabled rule")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty_rules", func(t *testing.T) {
|
||||||
|
if HasTagRules(nil) {
|
||||||
|
t.Error("expected false for nil rules")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
42
weed/s3api/s3lifecycle/version_time.go
Normal file
42
weed/s3api/s3lifecycle/version_time.go
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// versionIdFormatThreshold distinguishes old vs new format version IDs.
|
||||||
|
// New format (inverted timestamps) produces values above this threshold;
|
||||||
|
// old format (raw timestamps) produces values below it.
|
||||||
|
const versionIdFormatThreshold = 0x4000000000000000
|
||||||
|
|
||||||
|
// GetVersionTimestamp extracts the actual timestamp from a SeaweedFS version ID,
|
||||||
|
// handling both old (raw nanosecond) and new (inverted nanosecond) formats.
|
||||||
|
// Returns zero time if the version ID is invalid or "null".
|
||||||
|
func GetVersionTimestamp(versionId string) time.Time {
|
||||||
|
ns := getVersionTimestampNanos(versionId)
|
||||||
|
if ns == 0 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return time.Unix(0, ns)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getVersionTimestampNanos extracts the raw nanosecond timestamp from a version ID.
|
||||||
|
func getVersionTimestampNanos(versionId string) int64 {
|
||||||
|
if len(versionId) < 16 || versionId == "null" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
timestampPart, err := strconv.ParseUint(versionId[:16], 16, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if timestampPart > math.MaxInt64 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if timestampPart > versionIdFormatThreshold {
|
||||||
|
// New format: inverted timestamp, convert back.
|
||||||
|
return int64(math.MaxInt64 - timestampPart)
|
||||||
|
}
|
||||||
|
return int64(timestampPart)
|
||||||
|
}
|
||||||
74
weed/s3api/s3lifecycle/version_time_test.go
Normal file
74
weed/s3api/s3lifecycle/version_time_test.go
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
package s3lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetVersionTimestamp(t *testing.T) {
|
||||||
|
t.Run("new_format_inverted_timestamp", func(t *testing.T) {
|
||||||
|
// Simulate a new-format version ID (inverted timestamp above threshold).
|
||||||
|
now := time.Now()
|
||||||
|
inverted := math.MaxInt64 - now.UnixNano()
|
||||||
|
versionId := fmt.Sprintf("%016x", inverted) + "0000000000000000"
|
||||||
|
|
||||||
|
got := GetVersionTimestamp(versionId)
|
||||||
|
// Should recover the original timestamp within 1 second.
|
||||||
|
diff := got.Sub(now)
|
||||||
|
if diff < -time.Second || diff > time.Second {
|
||||||
|
t.Errorf("timestamp diff too large: %v (got %v, want ~%v)", diff, got, now)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("old_format_raw_timestamp", func(t *testing.T) {
|
||||||
|
// Simulate an old-format version ID (raw nanosecond timestamp below threshold).
|
||||||
|
// Use a timestamp from 2023 which would be below threshold.
|
||||||
|
ts := time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC)
|
||||||
|
versionId := fmt.Sprintf("%016x", ts.UnixNano()) + "abcdef0123456789"
|
||||||
|
|
||||||
|
got := GetVersionTimestamp(versionId)
|
||||||
|
if !got.Equal(ts) {
|
||||||
|
t.Errorf("expected %v, got %v", ts, got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("null_version_id", func(t *testing.T) {
|
||||||
|
got := GetVersionTimestamp("null")
|
||||||
|
if !got.IsZero() {
|
||||||
|
t.Errorf("expected zero time for null version, got %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("empty_version_id", func(t *testing.T) {
|
||||||
|
got := GetVersionTimestamp("")
|
||||||
|
if !got.IsZero() {
|
||||||
|
t.Errorf("expected zero time for empty version, got %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("short_version_id", func(t *testing.T) {
|
||||||
|
got := GetVersionTimestamp("abc")
|
||||||
|
if !got.IsZero() {
|
||||||
|
t.Errorf("expected zero time for short version, got %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("high_bit_overflow_returns_zero", func(t *testing.T) {
|
||||||
|
// Version ID with first 16 hex chars > math.MaxInt64 should return zero,
|
||||||
|
// not a wrapped negative timestamp.
|
||||||
|
versionId := "80000000000000000000000000000000"
|
||||||
|
got := GetVersionTimestamp(versionId)
|
||||||
|
if !got.IsZero() {
|
||||||
|
t.Errorf("expected zero time for overflow version ID, got %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("invalid_hex", func(t *testing.T) {
|
||||||
|
got := GetVersionTimestamp("zzzzzzzzzzzzzzzz0000000000000000")
|
||||||
|
if !got.IsZero() {
|
||||||
|
t.Errorf("expected zero time for invalid hex, got %v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user