36c469e34e
* Enforce IAM for s3tables bucket creation
* Prefer IAM path when policies exist
* Ensure IAM enforcement honors default allow
* address comments
* Reused the precomputed principal when setting tableBucketMetadata.OwnerAccountID, avoiding the redundant getAccountID call.
* get identity
* fix
* dedup
* fix
* comments
* fix tests
* update iam config
* go fmt
* fix ports
* fix flags
* mini clean shutdown
* Revert "update iam config"
This reverts commit ca48fdbb0afa45657823d98657556c0bbf24f239.
Revert "mini clean shutdown"
This reverts commit 9e17f6baffd5dd7cc404d831d18dd618b9fe5049.
Revert "fix flags"
This reverts commit e9e7b29d2f77ee5cb82147d50621255410695ee3.
Revert "go fmt"
This reverts commit bd3241960b1d9484b7900190773b0ecb3f762c9a.
* test/s3tables: share single weed mini per test package via TestMain
Previously each top-level test function in the catalog and s3tables
package started and stopped its own weed mini instance. This caused
failures when a prior instance wasn't cleanly stopped before the next
one started (port conflicts, leaked global state).
Changes:
- catalog/iceberg_catalog_test.go: introduce TestMain that starts one
shared TestEnvironment (external weed binary) before all tests and
tears it down after. All individual test functions now use sharedEnv.
Added randomSuffix() for unique resource names across tests.
- catalog/pyiceberg_test.go: updated to use sharedEnv instead of
per-test environments.
- catalog/pyiceberg_test_helpers.go -> pyiceberg_test_helpers_test.go:
renamed to a _test.go file so it can access TestEnvironment which is
defined in a test file.
- table-buckets/setup.go: add package-level sharedCluster variable.
- table-buckets/s3tables_integration_test.go: introduce TestMain that
starts one shared TestCluster before all tests. TestS3TablesIntegration
now uses sharedCluster. Extract startMiniClusterInDir (no *testing.T)
for TestMain use. TestS3TablesCreateBucketIAMPolicy keeps its own
cluster (different IAM config). Remove miniClusterMutex (no longer
needed). Fix Stop() to not panic when t is nil."
* delete
* parse
* default allow should work with anonymous
* fix port
* iceberg route
The failures are from Iceberg REST using the default bucket warehouse when no prefix is provided. Your tests create random buckets, so /v1/namespaces was looking in warehouse and failing. I updated the tests to use the prefixed Iceberg routes (/v1/{bucket}/...) via a small helper.
* test(s3tables): fix port conflicts and IAM ARN matching in integration tests
- Pass -master.dir explicitly to prevent filer store directory collision
between shared cluster and per-test clusters running in the same process
- Pass -volume.port.public and -volume.publicUrl to prevent the global
publicPort flag (mutated from 0 → concrete port by first cluster) from
being reused by a second cluster, causing 'address already in use'
- Remove the flag-reset loop in Stop() that reset global flag values while
other goroutines were reading them (race → panic)
- Fix IAM policy Resource ARN in TestS3TablesCreateBucketIAMPolicy to use
wildcards (arn:aws:s3tables:*:*:bucket/<name>) because the handler
generates ARNs with its own DefaultRegion (us-east-1) and principal name
('admin'), not the test constants testRegion/testAccountID
138 lines
4.4 KiB
Go
138 lines
4.4 KiB
Go
// Package catalog provides integration tests for the Iceberg REST Catalog API.
|
|
// This file adds PyIceberg-based compatibility tests using Docker.
|
|
package catalog
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// TestPyIcebergRestCatalog tests the Iceberg REST Catalog using PyIceberg client in Docker.
|
|
// This provides a more comprehensive test than DuckDB as PyIceberg fully exercises the REST API.
|
|
//
|
|
// Prerequisites:
|
|
// - Docker must be available
|
|
// - SeaweedFS must be running with Iceberg REST enabled
|
|
//
|
|
// To run manually:
|
|
//
|
|
// cd test/s3tables/catalog
|
|
// docker compose -f docker-compose.test.yaml up --build
|
|
func TestPyIcebergRestCatalog(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test in short mode")
|
|
}
|
|
|
|
env := sharedEnv
|
|
|
|
if !env.dockerAvailable {
|
|
t.Skip("Docker not available, skipping PyIceberg integration test")
|
|
}
|
|
|
|
// Create the test bucket first
|
|
bucketName := "pyiceberg-compat-test-" + randomSuffix()
|
|
createTableBucket(t, env, bucketName)
|
|
|
|
// Build the test working directory path
|
|
testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
|
|
|
|
// Run PyIceberg test using Docker
|
|
catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
|
|
s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
|
warehouse := fmt.Sprintf("s3://%s/", bucketName)
|
|
|
|
// Build the test image first for faster repeated runs
|
|
buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
|
|
buildCmd.Dir = testDir
|
|
if out, err := buildCmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
|
|
}
|
|
|
|
cmd := exec.Command("docker", "run", "--rm",
|
|
"--add-host", "host.docker.internal:host-gateway",
|
|
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
|
|
"-v", fmt.Sprintf("%s:/app:ro", testDir),
|
|
"iceberg-rest-test",
|
|
"python3", "/app/test_rest_catalog.py",
|
|
"--catalog-url", catalogURL,
|
|
"--warehouse", warehouse,
|
|
"--prefix", bucketName,
|
|
)
|
|
cmd.Dir = testDir
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
|
|
t.Logf("Running PyIceberg REST catalog test...")
|
|
t.Logf(" Catalog URL: %s", catalogURL)
|
|
t.Logf(" Warehouse: %s", warehouse)
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
t.Errorf("PyIceberg test failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
|
|
// This test uses the default admin credentials that SeaweedFS creates on startup.
|
|
func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test in short mode")
|
|
}
|
|
|
|
env := sharedEnv
|
|
|
|
if !env.dockerAvailable {
|
|
t.Skip("Docker not available, skipping PyIceberg integration test")
|
|
}
|
|
|
|
// Use default admin credentials
|
|
testAccessKey := "admin"
|
|
testSecretKey := "admin"
|
|
|
|
// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
|
|
bucketName := "pyiceberg-auth-test-" + randomSuffix()
|
|
createTableBucket(t, env, bucketName)
|
|
|
|
// Build the test working directory path
|
|
testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
|
|
|
|
// Run PyIceberg test using Docker with authentication
|
|
catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
|
|
s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
|
warehouse := fmt.Sprintf("s3://%s/", bucketName)
|
|
|
|
// Build the test image first for faster repeated runs
|
|
buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
|
|
buildCmd.Dir = testDir
|
|
if out, err := buildCmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
|
|
}
|
|
|
|
cmd := exec.Command("docker", "run", "--rm",
|
|
"--add-host", "host.docker.internal:host-gateway",
|
|
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
|
|
"-v", fmt.Sprintf("%s:/app:ro", testDir),
|
|
"iceberg-rest-test",
|
|
"python3", "/app/test_rest_catalog_auth.py",
|
|
"--catalog-url", catalogURL,
|
|
"--warehouse", warehouse,
|
|
"--prefix", bucketName,
|
|
"--access-key", testAccessKey,
|
|
"--secret-key", testSecretKey,
|
|
)
|
|
cmd.Dir = testDir
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
|
|
t.Logf("Running PyIceberg REST catalog test with authentication...")
|
|
t.Logf(" Catalog URL: %s", catalogURL)
|
|
t.Logf(" Warehouse: %s", warehouse)
|
|
t.Logf(" Access Key: %s", testAccessKey)
|
|
|
|
if err := cmd.Run(); err != nil {
|
|
t.Errorf("PyIceberg authenticated test failed: %v", err)
|
|
}
|
|
}
|