gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

iceberg: add delete file rewrite maintenance (#8664)

Browse Source 
* iceberg: add delete file rewrite maintenance

* iceberg: preserve untouched delete files during rewrites

* iceberg: share detection threshold defaults

* iceberg: add partition-scoped maintenance filters (#8665)

* iceberg: add partition-scoped maintenance filters

* iceberg: tighten where-filter partition matching
This commit is contained in:
Chris Lu
2026-03-16 21:11:09 -07:00
committed by GitHub
parent a3717cd4b5
commit e5c0889473
11 changed files with 2138 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
weed/plugin/worker/iceberg/handler_test.go
View File

@@ -42,15 +42,17 @@ func TestParseOperations(t *testing.T) {
expected []string
wantErr bool
}{
{"all", []string{"compact", "expire_snapshots", "remove_orphans", "rewrite_manifests"}, false},
{"", []string{"compact", "expire_snapshots", "remove_orphans", "rewrite_manifests"}, false},
{"all", []string{"compact", "rewrite_position_delete_files", "expire_snapshots", "remove_orphans", "rewrite_manifests"}, false},
{"", []string{"compact", "rewrite_position_delete_files", "expire_snapshots", "remove_orphans", "rewrite_manifests"}, false},
{"expire_snapshots", []string{"expire_snapshots"}, false},
{"compact", []string{"compact"}, false},
{"rewrite_position_delete_files", []string{"rewrite_position_delete_files"}, false},
{"rewrite_manifests,expire_snapshots", []string{"expire_snapshots", "rewrite_manifests"}, false},
{"compact,expire_snapshots", []string{"compact", "expire_snapshots"}, false},
{"remove_orphans, rewrite_manifests", []string{"remove_orphans", "rewrite_manifests"}, false},
{"expire_snapshots,remove_orphans,rewrite_manifests", []string{"expire_snapshots", "remove_orphans", "rewrite_manifests"}, false},
{"compact,expire_snapshots,remove_orphans,rewrite_manifests", []string{"compact", "expire_snapshots", "remove_orphans", "rewrite_manifests"}, false},
{"compact,rewrite_position_delete_files,rewrite_manifests", []string{"compact", "rewrite_position_delete_files", "rewrite_manifests"}, false},
{"unknown_op", nil, true},
{"expire_snapshots,bad_op", nil, true},
}
@@ -848,6 +850,35 @@ func TestParseConfigApplyDeletes(t *testing.T) {
}
}
func TestNormalizeDetectionConfigUsesSharedDefaults(t *testing.T) {
config := normalizeDetectionConfig(Config{})
if config.TargetFileSizeBytes != defaultTargetFileSizeMB*1024*1024 {
t.Fatalf("expected TargetFileSizeBytes default, got %d", config.TargetFileSizeBytes)
}
if config.DeleteTargetFileSizeBytes != defaultDeleteTargetFileSizeMB*1024*1024 {
t.Fatalf("expected DeleteTargetFileSizeBytes default, got %d", config.DeleteTargetFileSizeBytes)
}
if config.DeleteMinInputFiles != defaultDeleteMinInputFiles {
t.Fatalf("expected DeleteMinInputFiles default, got %d", config.DeleteMinInputFiles)
}
if config.DeleteMaxFileGroupSizeBytes != defaultDeleteMaxGroupSizeMB*1024*1024 {
t.Fatalf("expected DeleteMaxFileGroupSizeBytes default, got %d", config.DeleteMaxFileGroupSizeBytes)
}
if config.DeleteMaxOutputFiles != defaultDeleteMaxOutputFiles {
t.Fatalf("expected DeleteMaxOutputFiles default, got %d", config.DeleteMaxOutputFiles)
}
if config.OrphanOlderThanHours != defaultOrphanOlderThanHours {
t.Fatalf("expected OrphanOlderThanHours default, got %d", config.OrphanOlderThanHours)
}
if config.SnapshotRetentionHours != defaultSnapshotRetentionHours {
t.Fatalf("expected SnapshotRetentionHours default, got %d", config.SnapshotRetentionHours)
}
if config.MaxSnapshotsToKeep != defaultMaxSnapshotsToKeep {
t.Fatalf("expected MaxSnapshotsToKeep default, got %d", config.MaxSnapshotsToKeep)
}
}
func TestCollectPositionDeletes(t *testing.T) {
fs, client := startFakeFiler(t)