iceberg: handle filer-backed compaction inputs (#8638)

* iceberg: handle filer-backed compaction inputs

* iceberg: preserve upsert creation times

* iceberg: align compaction test schema

* iceberg: tighten compact output assertion

* iceberg: document compact output match

* iceberg: clear stale chunks in upsert helper

* iceberg: strengthen compaction integration coverage
This commit is contained in:
Chris Lu
2026-03-15 17:46:06 -07:00
committed by GitHub
parent 0afc675a55
commit e24630251c
3 changed files with 358 additions and 9 deletions

View File

@@ -109,6 +109,22 @@ func (h *Handler) compactDataFiles(
var deletedManifestEntries []iceberg.ManifestEntry
totalMerged := 0
entrySeqNum := func(entry iceberg.ManifestEntry) *int64 {
seqNum := entry.SequenceNum()
if seqNum < 0 {
return nil
}
return &seqNum
}
entryFileSeqNum := func(entry iceberg.ManifestEntry) *int64 {
if fileSeqNum := entry.FileSequenceNum(); fileSeqNum != nil {
value := *fileSeqNum
return &value
}
return entrySeqNum(entry)
}
metaDir := path.Join(s3tables.TablesPath, bucketName, tablePath, "metadata")
dataDir := path.Join(s3tables.TablesPath, bucketName, tablePath, "data")
@@ -189,7 +205,7 @@ func (h *Handler) compactDataFiles(
delEntry := iceberg.NewManifestEntry(
iceberg.EntryStatusDELETED,
&newSnapID,
nil, nil,
entrySeqNum(entry), entryFileSeqNum(entry),
entry.DataFile(),
)
deletedManifestEntries = append(deletedManifestEntries, delEntry)
@@ -221,7 +237,7 @@ func (h *Handler) compactDataFiles(
existingEntry := iceberg.NewManifestEntry(
iceberg.EntryStatusEXISTING,
func() *int64 { id := entry.SnapshotID(); return &id }(),
nil, nil,
entrySeqNum(entry), entryFileSeqNum(entry),
entry.DataFile(),
)
manifestEntries = append(manifestEntries, existingEntry)