s3lifecycle: add lifecycle rule evaluator package and extend XML types (#8807)

* s3api: extend lifecycle XML types with NoncurrentVersionExpiration, AbortIncompleteMultipartUpload Add missing S3 lifecycle rule types to the XML data model: - NoncurrentVersionExpiration with NoncurrentDays and NewerNoncurrentVersions - NoncurrentVersionTransition with NoncurrentDays and StorageClass - AbortIncompleteMultipartUpload with DaysAfterInitiation - Filter.ObjectSizeGreaterThan and ObjectSizeLessThan - And.ObjectSizeGreaterThan and ObjectSizeLessThan - Filter.UnmarshalXML to properly parse Tag, And, and size filter elements Each new type follows the existing set-field pattern for conditional XML marshaling. No behavior changes - these types are not yet wired into handlers or the lifecycle worker. * s3lifecycle: add lifecycle rule evaluator package New package weed/s3api/s3lifecycle/ provides a pure-function lifecycle rule evaluation engine. The evaluator accepts flattened Rule structs and ObjectInfo metadata, and returns the appropriate Action. Components: - evaluator.go: Evaluate() for per-object actions with S3 priority ordering (delete marker > noncurrent version > current expiration), ShouldExpireNoncurrentVersion() with NewerNoncurrentVersions support, EvaluateMPUAbort() for multipart upload rules - filter.go: prefix, tag, and size-based filter matching - tags.go: ExtractTags() extracts S3 tags from filer Extended metadata, HasTagRules() for scan-time optimization - version_time.go: GetVersionTimestamp() extracts timestamps from SeaweedFS version IDs (both old and new format) Comprehensive test coverage: 54 tests covering all action types, filter combinations, edge cases, and version ID formats. * s3api: add UnmarshalXML for Expiration, Transition, ExpireDeleteMarker Add UnmarshalXML methods that set the internal 'set' flag during XML parsing. Previously these flags were only set programmatically, causing XML round-trip to drop elements. This ensures lifecycle configurations stored as XML survive unmarshal/marshal cycles correctly. Add comprehensive XML round-trip tests for all lifecycle rule types including NoncurrentVersionExpiration, AbortIncompleteMultipartUpload, Filter with Tag/And/size constraints, and a complete Terraform-style lifecycle configuration. * s3lifecycle: address review feedback - Fix version_time.go overflow: guard timestampPart > MaxInt64 before the inversion subtraction to prevent uint64 wrap - Make all expiry checks inclusive (!now.Before instead of now.After) so actions trigger at the exact scheduled instant - Add NoncurrentIndex to ObjectInfo so Evaluate() can properly handle NewerNoncurrentVersions via ShouldExpireNoncurrentVersion() - Add test for high-bit overflow version ID * s3lifecycle: guard ShouldExpireNoncurrentVersion against zero SuccessorModTime Add early return when obj.IsLatest or obj.SuccessorModTime.IsZero() to prevent premature expiration of versions with uninitialized successor timestamps (zero value would compute to epoch, always expired). --------- Co-authored-by: Copilot <copilot@github.com>
2026-03-28 11:10:31 -07:00
parent 7d5cbfd547
commit 54dd4f091d
11 changed files with 1523 additions and 12 deletions
--- a/weed/s3api/s3api_policy.go
+++ b/weed/s3api/s3api_policy.go
@@ -22,13 +22,16 @@ type Lifecycle struct {

 // Rule - a rule for lifecycle configuration.
 type Rule struct {
-	XMLName    xml.Name   `xml:"Rule"`
-	ID         string     `xml:"ID,omitempty"`
-	Status     ruleStatus `xml:"Status"`
-	Filter     Filter     `xml:"Filter,omitempty"`
-	Prefix     Prefix     `xml:"Prefix,omitempty"`
-	Expiration Expiration `xml:"Expiration,omitempty"`
-	Transition Transition `xml:"Transition,omitempty"`
+	XMLName                        xml.Name                       `xml:"Rule"`
+	ID                             string                         `xml:"ID,omitempty"`
+	Status                         ruleStatus                     `xml:"Status"`
+	Filter                         Filter                         `xml:"Filter,omitempty"`
+	Prefix                         Prefix                         `xml:"Prefix,omitempty"`
+	Expiration                     Expiration                     `xml:"Expiration,omitempty"`
+	Transition                     Transition                     `xml:"Transition,omitempty"`
+	NoncurrentVersionExpiration    NoncurrentVersionExpiration    `xml:"NoncurrentVersionExpiration,omitempty"`
+	NoncurrentVersionTransition    NoncurrentVersionTransition    `xml:"NoncurrentVersionTransition,omitempty"`
+	AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
 }

 // Filter - a filter for a lifecycle configuration Rule.
@@ -43,6 +46,9 @@ type Filter struct {

 	Tag    Tag
 	tagSet bool
+
+	ObjectSizeGreaterThan int64
+	ObjectSizeLessThan    int64
 }

 // Prefix holds the prefix xml tag in <Rule> and <Filter>
@@ -80,17 +86,83 @@ func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 	if err := e.EncodeToken(start); err != nil {
 		return err
 	}
-	if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil {
-		return err
+	if f.andSet {
+		if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil {
+			return err
+		}
+	} else if f.tagSet {
+		if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil {
+			return err
+		}
+	} else {
+		if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil {
+			return err
+		}
+	}
+	if f.ObjectSizeGreaterThan > 0 {
+		if err := e.EncodeElement(f.ObjectSizeGreaterThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeGreaterThan"}}); err != nil {
+			return err
+		}
+	}
+	if f.ObjectSizeLessThan > 0 {
+		if err := e.EncodeElement(f.ObjectSizeLessThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeLessThan"}}); err != nil {
+			return err
+		}
 	}
 	return e.EncodeToken(xml.EndElement{Name: start.Name})
 }

+// UnmarshalXML decodes Filter from XML, handling all child elements.
+func (f *Filter) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	f.set = true
+	for {
+		tok, err := d.Token()
+		if err != nil {
+			return err
+		}
+		switch t := tok.(type) {
+		case xml.StartElement:
+			switch t.Name.Local {
+			case "Prefix":
+				if err := d.DecodeElement(&f.Prefix, &t); err != nil {
+					return err
+				}
+			case "Tag":
+				f.tagSet = true
+				if err := d.DecodeElement(&f.Tag, &t); err != nil {
+					return err
+				}
+			case "And":
+				f.andSet = true
+				if err := d.DecodeElement(&f.And, &t); err != nil {
+					return err
+				}
+			case "ObjectSizeGreaterThan":
+				if err := d.DecodeElement(&f.ObjectSizeGreaterThan, &t); err != nil {
+					return err
+				}
+			case "ObjectSizeLessThan":
+				if err := d.DecodeElement(&f.ObjectSizeLessThan, &t); err != nil {
+					return err
+				}
+			default:
+				if err := d.Skip(); err != nil {
+					return err
+				}
+			}
+		case xml.EndElement:
+			return nil
+		}
+	}
+}
+
 // And - a tag to combine a prefix and multiple tags for lifecycle configuration rule.
 type And struct {
-	XMLName xml.Name `xml:"And"`
-	Prefix  Prefix   `xml:"Prefix,omitempty"`
-	Tags    []Tag    `xml:"Tag,omitempty"`
+	XMLName               xml.Name `xml:"And"`
+	Prefix                Prefix   `xml:"Prefix,omitempty"`
+	Tags                  []Tag    `xml:"Tag,omitempty"`
+	ObjectSizeGreaterThan int64    `xml:"ObjectSizeGreaterThan,omitempty"`
+	ObjectSizeLessThan    int64    `xml:"ObjectSizeLessThan,omitempty"`
 }

 // Expiration - expiration actions for a rule in lifecycle configuration.
@@ -112,6 +184,17 @@ func (e Expiration) MarshalXML(enc *xml.Encoder, startElement xml.StartElement)
 	return enc.EncodeElement(expirationWrapper(e), startElement)
 }

+func (e *Expiration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	type wrapper Expiration
+	var w wrapper
+	if err := d.DecodeElement(&w, &start); err != nil {
+		return err
+	}
+	*e = Expiration(w)
+	e.set = true
+	return nil
+}
+
 // ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element.
 type ExpireDeleteMarker struct {
 	val bool
@@ -126,6 +209,15 @@ func (b ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartEle
 	return e.EncodeElement(b.val, startElement)
 }

+func (b *ExpireDeleteMarker) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	var v bool
+	if err := d.DecodeElement(&v, &start); err != nil {
+		return err
+	}
+	*b = ExpireDeleteMarker{val: v, set: true}
+	return nil
+}
+
 // ExpirationDate is a embedded type containing time.Time to unmarshal
 // Date in Expiration
 type ExpirationDate struct {
@@ -160,5 +252,102 @@ func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
 	return enc.EncodeElement(transitionWrapper(t), start)
 }

+func (t *Transition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	type wrapper Transition
+	var w wrapper
+	if err := d.DecodeElement(&w, &start); err != nil {
+		return err
+	}
+	*t = Transition(w)
+	t.set = true
+	return nil
+}
+
 // TransitionDays is a type alias to unmarshal Days in Transition
 type TransitionDays int
+
+// NoncurrentVersionExpiration - expiration actions for non-current object versions.
+type NoncurrentVersionExpiration struct {
+	XMLName                 xml.Name `xml:"NoncurrentVersionExpiration"`
+	NoncurrentDays          int      `xml:"NoncurrentDays,omitempty"`
+	NewerNoncurrentVersions int      `xml:"NewerNoncurrentVersions,omitempty"`
+
+	set bool
+}
+
+// MarshalXML encodes NoncurrentVersionExpiration field into an XML form.
+func (n NoncurrentVersionExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	if !n.set {
+		return nil
+	}
+	type wrapper NoncurrentVersionExpiration
+	return enc.EncodeElement(wrapper(n), start)
+}
+
+func (n *NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	type wrapper NoncurrentVersionExpiration
+	var w wrapper
+	if err := d.DecodeElement(&w, &start); err != nil {
+		return err
+	}
+	*n = NoncurrentVersionExpiration(w)
+	n.set = true
+	return nil
+}
+
+// NoncurrentVersionTransition - transition actions for non-current object versions.
+type NoncurrentVersionTransition struct {
+	XMLName        xml.Name `xml:"NoncurrentVersionTransition"`
+	NoncurrentDays int      `xml:"NoncurrentDays,omitempty"`
+	StorageClass   string   `xml:"StorageClass,omitempty"`
+
+	set bool
+}
+
+// MarshalXML encodes NoncurrentVersionTransition field into an XML form.
+func (n NoncurrentVersionTransition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	if !n.set {
+		return nil
+	}
+	type wrapper NoncurrentVersionTransition
+	return enc.EncodeElement(wrapper(n), start)
+}
+
+func (n *NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	type wrapper NoncurrentVersionTransition
+	var w wrapper
+	if err := d.DecodeElement(&w, &start); err != nil {
+		return err
+	}
+	*n = NoncurrentVersionTransition(w)
+	n.set = true
+	return nil
+}
+
+// AbortIncompleteMultipartUpload - abort action for incomplete multipart uploads.
+type AbortIncompleteMultipartUpload struct {
+	XMLName             xml.Name `xml:"AbortIncompleteMultipartUpload"`
+	DaysAfterInitiation int      `xml:"DaysAfterInitiation,omitempty"`
+
+	set bool
+}
+
+// MarshalXML encodes AbortIncompleteMultipartUpload field into an XML form.
+func (a AbortIncompleteMultipartUpload) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
+	if !a.set {
+		return nil
+	}
+	type wrapper AbortIncompleteMultipartUpload
+	return enc.EncodeElement(wrapper(a), start)
+}
+
+func (a *AbortIncompleteMultipartUpload) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	type wrapper AbortIncompleteMultipartUpload
+	var w wrapper
+	if err := d.DecodeElement(&w, &start); err != nil {
+		return err
+	}
+	*a = AbortIncompleteMultipartUpload(w)
+	a.set = true
+	return nil
+}