Add Apache Polaris integration tests (#8478)

* test: add polaris integration test harness

* test: add polaris integration coverage

* ci: run polaris s3 tables tests

* test: harden polaris harness

* test: DRY polaris integration tests

* ci: pre-pull Polaris image

* test: extend Polaris pull timeout

* test: refine polaris credentials selection

* test: keep Polaris tables inside allowed location

* test: use fresh context for polaris cleanup

* test: prefer specific Polaris storage credential

* test: tolerate Polaris credential variants

* test: request Polaris vended credentials

* test: load Polaris table credentials

* test: allow polaris vended access via bucket policy

* test: align Polaris object keys with table location

.github/workflows/s3-tables-tests.yml

@@ -194,6 +194,67 @@ jobs:
           path: test/s3tables/catalog_trino/test-output.log
           retention-days: 3
 
+  polaris-integration-tests:
+    name: Polaris Integration Tests
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v6
+
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Run go mod tidy
+        run: go mod tidy
+
+      - name: Install SeaweedFS
+        run: |
+          go install -buildvcs=false ./weed
+
+      - name: Pre-pull Polaris image
+        run: docker pull apache/polaris:latest
+
+      - name: Run Polaris Integration Tests
+        timeout-minutes: 25
+        run: |
+          set -x
+          set -o pipefail
+          echo "=== System Information ==="
+          uname -a
+          free -h
+          df -h
+          echo "=== Starting Polaris Tests ==="
+
+          go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
+            echo "Polaris integration tests failed"
+            exit 1
+          }
+
+      - name: Show test output on failure
+        if: failure()
+        working-directory: test/s3tables/polaris
+        run: |
+          echo "=== Test Output ==="
+          if [ -f test-output.log ]; then
+            tail -200 test-output.log
+          fi
+
+          echo "=== Process information ==="
+          ps aux | grep -E "(weed|test|docker)" || true
+
+      - name: Upload test logs on failure
+        if: failure()
+        uses: actions/upload-artifact@v6
+        with:
+          name: polaris-test-logs
+          path: test/s3tables/polaris/test-output.log
+          retention-days: 3
+
   spark-iceberg-catalog-tests:
     name: Spark Iceberg Catalog Integration Tests
     runs-on: ubuntu-22.04

test/s3tables/polaris/polaris_env_test.go

@@ -0,0 +1,377 @@
+package polaris
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+
+	"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
+)
+
+const (
+	polarisImage            = "apache/polaris:latest"
+	polarisRealm            = "POLARIS"
+	polarisRootClientID     = "root"
+	polarisRootClientSecret = "s3cr3t"
+	polarisRegion           = "us-east-1"
+	polarisRoleArn          = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
+	polarisSigningKey       = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
+)
+
+type TestEnvironment struct {
+	seaweedDir       string
+	weedBinary       string
+	dataDir          string
+	bindIP           string
+	s3Port           int
+	s3GrpcPort       int
+	masterPort       int
+	masterGrpcPort   int
+	filerPort        int
+	filerGrpcPort    int
+	volumePort       int
+	volumeGrpcPort   int
+	polarisPort      int
+	polarisAdminPort int
+	weedProcess      *exec.Cmd
+	weedCancel       context.CancelFunc
+	polarisContainer string
+	accessKey        string
+	secretKey        string
+}
+
+func NewTestEnvironment(t *testing.T) *TestEnvironment {
+	t.Helper()
+
+	if !testutil.HasDocker() {
+		t.Skip("Docker is required for Polaris integration tests")
+	}
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	seaweedDir := wd
+	for i := 0; i < 6; i++ {
+		if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
+			break
+		}
+		seaweedDir = filepath.Dir(seaweedDir)
+	}
+
+	weedBinary := filepath.Join(seaweedDir, "weed", "weed")
+	if _, err := os.Stat(weedBinary); err != nil {
+		weedBinary = "weed"
+		if _, err := exec.LookPath(weedBinary); err != nil {
+			t.Skip("weed binary not found, skipping integration test")
+		}
+	}
+
+	dataDir, err := os.MkdirTemp("", "seaweed-polaris-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+
+	bindIP := testutil.FindBindIP()
+
+	masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
+	volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
+	filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
+	s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
+	polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
+
+	return &TestEnvironment{
+		seaweedDir:       seaweedDir,
+		weedBinary:       weedBinary,
+		dataDir:          dataDir,
+		bindIP:           bindIP,
+		s3Port:           s3Port,
+		s3GrpcPort:       s3GrpcPort,
+		masterPort:       masterPort,
+		masterGrpcPort:   masterGrpcPort,
+		filerPort:        filerPort,
+		filerGrpcPort:    filerGrpcPort,
+		volumePort:       volumePort,
+		volumeGrpcPort:   volumeGrpcPort,
+		polarisPort:      polarisPort,
+		polarisAdminPort: polarisAdminPort,
+		accessKey:        "admin",
+		secretKey:        "admin",
+	}
+}
+
+func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
+	t.Helper()
+
+	iamConfigPath := filepath.Join(env.dataDir, "iam.json")
+	iamConfig := fmt.Sprintf(`{
+  "identities": [
+    {
+      "name": "admin",
+      "credentials": [
+        {
+          "accessKey": "%s",
+          "secretKey": "%s"
+        }
+      ],
+      "actions": ["Admin", "Read", "List", "Tagging", "Write"]
+    }
+  ],
+  "sts": {
+    "tokenDuration": "12h",
+    "maxSessionLength": "24h",
+    "issuer": "seaweedfs-sts",
+    "signingKey": "%s"
+  },
+  "roles": [
+    {
+      "roleName": "LakekeeperVendedRole",
+      "roleArn": "%s",
+      "trustPolicy": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Principal": "*",
+            "Action": "sts:AssumeRole"
+          }
+        ]
+      },
+      "attachedPolicies": ["FullAccess"]
+    }
+  ],
+  "policies": [
+    {
+      "name": "FullAccess",
+      "document": {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": "*",
+            "Resource": "*"
+          }
+        ]
+      }
+    }
+  ]
+}`, env.accessKey, env.secretKey, polarisSigningKey, polarisRoleArn)
+
+	if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
+		t.Fatalf("Failed to create IAM config: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	env.weedCancel = cancel
+
+	cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
+		"-master.port", fmt.Sprintf("%d", env.masterPort),
+		"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
+		"-volume.port", fmt.Sprintf("%d", env.volumePort),
+		"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
+		"-filer.port", fmt.Sprintf("%d", env.filerPort),
+		"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
+		"-s3.port", fmt.Sprintf("%d", env.s3Port),
+		"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
+		"-s3.config", iamConfigPath,
+		"-s3.iam.config", iamConfigPath,
+		"-s3.iam.readOnly=false",
+		"-ip", env.bindIP,
+		"-ip.bind", "0.0.0.0",
+		"-dir", env.dataDir,
+	)
+	cmd.Dir = env.dataDir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		t.Fatalf("Failed to start SeaweedFS: %v", err)
+	}
+	env.weedProcess = cmd
+
+	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
+		t.Fatalf("S3 API failed to become ready")
+	}
+}
+
+func (env *TestEnvironment) StartPolaris(t *testing.T) {
+	t.Helper()
+
+	containerName := fmt.Sprintf("seaweed-polaris-%d", time.Now().UnixNano())
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "docker", "run", "-d", "--rm",
+		"--name", containerName,
+		"--add-host", "host.docker.internal:host-gateway",
+		"-p", fmt.Sprintf("%d:8181", env.polarisPort),
+		"-p", fmt.Sprintf("%d:8182", env.polarisAdminPort),
+		"-e", fmt.Sprintf("AWS_REGION=%s", polarisRegion),
+		"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", env.accessKey),
+		"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", env.secretKey),
+		"-e", fmt.Sprintf("POLARIS_BOOTSTRAP_CREDENTIALS=%s,%s,%s", polarisRealm, polarisRootClientID, polarisRootClientSecret),
+		"-e", fmt.Sprintf("polaris.realm-context.realms=%s", polarisRealm),
+		"-e", "quarkus.otel.sdk.disabled=true",
+		polarisImage,
+	)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		if ctx.Err() == context.DeadlineExceeded {
+			t.Fatalf("Timed out waiting for Polaris container: %v\nOutput:\n%s", ctx.Err(), string(output))
+		}
+		t.Fatalf("Failed to start Polaris: %v\nOutput:\n%s", err, string(output))
+	}
+	env.polarisContainer = containerName
+
+	if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/q/health", env.polarisAdminPort), 60*time.Second) {
+		logCtx, logCancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer logCancel()
+		logs, _ := exec.CommandContext(logCtx, "docker", "logs", env.polarisContainer).CombinedOutput()
+		t.Fatalf("Polaris failed to become ready\nLogs:\n%s", string(logs))
+	}
+}
+
+func (env *TestEnvironment) Cleanup(t *testing.T) {
+	t.Helper()
+	if env.weedCancel != nil {
+		env.weedCancel()
+	}
+	if env.weedProcess != nil {
+		time.Sleep(1 * time.Second)
+		_ = env.weedProcess.Wait()
+	}
+	if env.polarisContainer != "" {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		_ = exec.CommandContext(ctx, "docker", "rm", "-f", env.polarisContainer).Run()
+	}
+	if env.dataDir != "" {
+		_ = os.RemoveAll(env.dataDir)
+	}
+}
+
+func (env *TestEnvironment) polarisEndpoint() string {
+	return fmt.Sprintf("http://127.0.0.1:%d", env.polarisPort)
+}
+
+func (env *TestEnvironment) s3Endpoint() string {
+	return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
+}
+
+func (env *TestEnvironment) s3InternalEndpoint() string {
+	return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
+}
+
+type polarisHTTPClient struct {
+	baseURL    string
+	realm      string
+	token      string
+	httpClient *http.Client
+}
+
+func newPolarisHTTPClient(baseURL, realm, token string) *polarisHTTPClient {
+	return &polarisHTTPClient{
+		baseURL: baseURL,
+		realm:   realm,
+		token:   token,
+		httpClient: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+	}
+}
+
+func (c *polarisHTTPClient) doJSON(ctx context.Context, method, path string, body interface{}, out interface{}) error {
+	return c.doJSONWithHeaders(ctx, method, path, body, out, nil)
+}
+
+func (c *polarisHTTPClient) doJSONWithHeaders(ctx context.Context, method, path string, body interface{}, out interface{}, headers map[string]string) error {
+	var reader io.Reader
+	if body != nil {
+		encoded, err := json.Marshal(body)
+		if err != nil {
+			return fmt.Errorf("encode request body: %w", err)
+		}
+		reader = bytes.NewReader(encoded)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
+	if err != nil {
+		return fmt.Errorf("create request: %w", err)
+	}
+	if body != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	req.Header.Set("Accept", "application/json")
+	if c.realm != "" {
+		req.Header.Set("Polaris-Realm", c.realm)
+	}
+	if c.token != "" {
+		req.Header.Set("Authorization", "Bearer "+c.token)
+	}
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		bodyBytes, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			return fmt.Errorf("request failed with status %d and could not read response body: %w", resp.StatusCode, readErr)
+		}
+		return fmt.Errorf("request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
+	}
+	if out == nil {
+		return nil
+	}
+	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	return nil
+}
+
+func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials, pathStyle bool) (*s3.Client, error) {
+	resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+		if service == s3.ServiceID {
+			return aws.Endpoint{
+				URL:               endpoint,
+				SigningRegion:     region,
+				HostnameImmutable: true,
+			}, nil
+		}
+		return aws.Endpoint{}, &aws.EndpointNotFoundError{}
+	})
+
+	cfg, err := config.LoadDefaultConfig(ctx,
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
+		config.WithEndpointResolverWithOptions(resolver),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = pathStyle
+	}), nil
+}

test/s3tables/polaris/polaris_test.go

@@ -0,0 +1,805 @@
+package polaris
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+type polarisSession struct {
+	catalogName  string
+	bucketName   string
+	token        string
+	baseLocation string
+}
+
+type polarisTableSetup struct {
+	namespace     string
+	table         string
+	dataKeyPrefix string
+	s3Client      *s3.Client
+}
+
+type polarisCatalogClient struct {
+	http    *polarisHTTPClient
+	catalog string
+}
+
+type polarisCredentials struct {
+	ClientID     string `json:"clientId"`
+	ClientSecret string `json:"clientSecret"`
+}
+
+type createPrincipalResponse struct {
+	Credentials polarisCredentials `json:"credentials"`
+}
+
+type createCatalogRequest struct {
+	Catalog polarisCatalog `json:"catalog"`
+}
+
+type polarisCatalog struct {
+	Name              string               `json:"name"`
+	Type              string               `json:"type"`
+	ReadOnly          bool                 `json:"readOnly"`
+	Properties        map[string]string    `json:"properties"`
+	StorageConfigInfo polarisStorageConfig `json:"storageConfigInfo"`
+}
+
+type polarisStorageConfig struct {
+	StorageType      string   `json:"storageType"`
+	AllowedLocations []string `json:"allowedLocations"`
+	Endpoint         string   `json:"endpoint"`
+	EndpointInternal string   `json:"endpointInternal"`
+	StsEndpoint      string   `json:"stsEndpoint"`
+	PathStyleAccess  bool     `json:"pathStyleAccess"`
+	RoleArn          string   `json:"roleArn"`
+	Region           string   `json:"region"`
+}
+
+type createNamespaceRequest struct {
+	Namespace []string `json:"namespace"`
+}
+
+type createTableRequest struct {
+	Name          string            `json:"name"`
+	Location      string            `json:"location"`
+	Schema        icebergSchema     `json:"schema"`
+	PartitionSpec icebergPartition  `json:"partition-spec"`
+	SortOrder     icebergSortOrder  `json:"sort-order"`
+	Properties    map[string]string `json:"properties"`
+}
+
+type icebergSchema struct {
+	Type     string               `json:"type"`
+	SchemaID int                  `json:"schema-id"`
+	Fields   []icebergSchemaField `json:"fields"`
+}
+
+type icebergSchemaField struct {
+	ID       int    `json:"id"`
+	Name     string `json:"name"`
+	Type     string `json:"type"`
+	Required bool   `json:"required"`
+}
+
+type icebergPartition struct {
+	SpecID int                     `json:"spec-id"`
+	Fields []icebergPartitionField `json:"fields"`
+}
+
+type icebergPartitionField struct {
+	SourceID  int    `json:"source-id"`
+	FieldID   int    `json:"field-id"`
+	Name      string `json:"name"`
+	Transform string `json:"transform"`
+}
+
+type icebergSortOrder struct {
+	OrderID int                `json:"order-id"`
+	Fields  []icebergSortField `json:"fields"`
+}
+
+type icebergSortField struct {
+	SourceID  int    `json:"source-id"`
+	Direction string `json:"direction"`
+	NullOrder string `json:"null-order"`
+}
+
+type loadTableResponse struct {
+	Config             map[string]string   `json:"config"`
+	StorageCredentials []storageCredential `json:"storage-credentials"`
+}
+
+type loadCredentialsResponse struct {
+	StorageCredentials []storageCredential `json:"storage-credentials"`
+}
+
+type storageCredential struct {
+	Prefix string            `json:"prefix"`
+	Config map[string]string `json:"config"`
+}
+
+func bootstrapPolarisTest(t *testing.T, env *TestEnvironment) (context.Context, context.CancelFunc, polarisSession, *polarisTableSetup, func()) {
+	t.Helper()
+
+	t.Logf(">>> Starting SeaweedFS with Polaris configuration...")
+	env.StartSeaweedFS(t)
+	t.Logf(">>> SeaweedFS started.")
+
+	t.Logf(">>> Starting Polaris...")
+	env.StartPolaris(t)
+	t.Logf(">>> Polaris started.")
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	session := newPolarisSession(t, ctx, env)
+	setup, cleanup := setupPolarisTable(t, ctx, env, session)
+
+	return ctx, cancel, session, setup, cleanup
+}
+
+func TestPolarisIntegration(t *testing.T) {
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
+	defer cancel()
+	defer cleanup()
+
+	objectKey := fmt.Sprintf("%s/hello-%d.txt", setup.dataKeyPrefix, time.Now().UnixNano())
+	payload := []byte("polaris")
+
+	if _, err := setup.s3Client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+		Body:   bytes.NewReader(payload),
+	}); err != nil {
+		t.Fatalf("PutObject failed: %v", err)
+	}
+
+	listObjects, err := setup.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+		Bucket: aws.String(session.bucketName),
+		Prefix: aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("ListObjectsV2 failed: %v", err)
+	}
+
+	found := false
+	for _, obj := range listObjects.Contents {
+		if aws.ToString(obj.Key) == objectKey {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("Object %s not found in list", objectKey)
+	}
+
+	getResp, err := setup.s3Client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("GetObject failed: %v", err)
+	}
+	body, err := io.ReadAll(getResp.Body)
+	_ = getResp.Body.Close()
+	if err != nil {
+		t.Fatalf("Read object body failed: %v", err)
+	}
+	if !bytes.Equal(body, payload) {
+		t.Fatalf("Unexpected object payload: got %q want %q", string(body), string(payload))
+	}
+
+	if _, err := setup.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	}); err != nil {
+		t.Fatalf("DeleteObject failed: %v", err)
+	}
+}
+
+func TestPolarisTableIntegration(t *testing.T) {
+	env := NewTestEnvironment(t)
+	defer env.Cleanup(t)
+
+	ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
+	defer cancel()
+	defer cleanup()
+
+	objectKey := fmt.Sprintf("%s/part-%d.parquet", setup.dataKeyPrefix, time.Now().UnixNano())
+	createResp, err := setup.s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("CreateMultipartUpload failed: %v", err)
+	}
+
+	uploadID := aws.ToString(createResp.UploadId)
+	multipartCompleted := false
+	defer func() {
+		if uploadID == "" || multipartCompleted {
+			return
+		}
+		_, _ = setup.s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+			Bucket:   aws.String(session.bucketName),
+			Key:      aws.String(objectKey),
+			UploadId: aws.String(uploadID),
+		})
+	}()
+
+	partSize := 5 * 1024 * 1024
+	part1 := bytes.Repeat([]byte("a"), partSize)
+	part2 := bytes.Repeat([]byte("b"), 1024*1024)
+
+	part1Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(session.bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(1),
+		Body:       bytes.NewReader(part1),
+	})
+	if err != nil {
+		t.Fatalf("UploadPart 1 failed: %v", err)
+	}
+
+	part2Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
+		Bucket:     aws.String(session.bucketName),
+		Key:        aws.String(objectKey),
+		UploadId:   aws.String(uploadID),
+		PartNumber: aws.Int32(2),
+		Body:       bytes.NewReader(part2),
+	})
+	if err != nil {
+		t.Fatalf("UploadPart 2 failed: %v", err)
+	}
+
+	_, err = setup.s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+		Bucket:   aws.String(session.bucketName),
+		Key:      aws.String(objectKey),
+		UploadId: aws.String(uploadID),
+		MultipartUpload: &s3types.CompletedMultipartUpload{
+			Parts: []s3types.CompletedPart{
+				{
+					ETag:       part1Resp.ETag,
+					PartNumber: aws.Int32(1),
+				},
+				{
+					ETag:       part2Resp.ETag,
+					PartNumber: aws.Int32(2),
+				},
+			},
+		},
+	})
+	if err != nil {
+		t.Fatalf("CompleteMultipartUpload failed: %v", err)
+	}
+	multipartCompleted = true
+
+	headResp, err := setup.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(session.bucketName),
+		Key:    aws.String(objectKey),
+	})
+	if err != nil {
+		t.Fatalf("HeadObject after multipart upload failed: %v", err)
+	}
+	expectedSize := int64(len(part1) + len(part2))
+	if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
+		t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
+	}
+}
+
+func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) polarisSession {
+	t.Helper()
+
+	adminCreds := aws.Credentials{
+		AccessKeyID:     env.accessKey,
+		SecretAccessKey: env.secretKey,
+		Source:          "polaris-admin",
+	}
+	adminS3, err := newS3Client(ctx, env.s3Endpoint(), polarisRegion, adminCreds, true)
+	if err != nil {
+		t.Fatalf("Create admin S3 client failed: %v", err)
+	}
+
+	bucketName := fmt.Sprintf("polaris-bucket-%d", time.Now().UnixNano())
+	if _, err := adminS3.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+	}); err != nil {
+		t.Fatalf("CreateBucket failed: %v", err)
+	}
+	policy := fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Sid":"AllowPolarisVendedAccess","Effect":"Allow","Principal":"*","Action":"s3:*","Resource":["arn:aws:s3:::%s","arn:aws:s3:::%s/polaris/*"]}]}`, bucketName, bucketName)
+	if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(policy),
+	}); err != nil {
+		t.Fatalf("PutBucketPolicy failed: %v", err)
+	}
+
+	rootToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), polarisRootClientID, polarisRootClientSecret)
+	if err != nil {
+		t.Fatalf("Polaris root token request failed: %v", err)
+	}
+
+	managementClient := newPolarisHTTPClient(env.polarisEndpoint(), polarisRealm, rootToken)
+	catalogName := fmt.Sprintf("polaris_catalog_%d", time.Now().UnixNano())
+	baseLocation := fmt.Sprintf("s3://%s/polaris", bucketName)
+
+	catalogRequest := createCatalogRequest{
+		Catalog: polarisCatalog{
+			Name:     catalogName,
+			Type:     "INTERNAL",
+			ReadOnly: false,
+			Properties: map[string]string{
+				"default-base-location": baseLocation,
+			},
+			StorageConfigInfo: polarisStorageConfig{
+				StorageType:      "S3",
+				AllowedLocations: []string{baseLocation},
+				Endpoint:         env.s3Endpoint(),
+				EndpointInternal: env.s3InternalEndpoint(),
+				StsEndpoint:      env.s3InternalEndpoint(),
+				PathStyleAccess:  true,
+				RoleArn:          polarisRoleArn,
+				Region:           polarisRegion,
+			},
+		},
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/catalogs", catalogRequest, nil); err != nil {
+		t.Fatalf("Create catalog failed: %v", err)
+	}
+
+	principalName := fmt.Sprintf("polaris_user_%d", time.Now().UnixNano())
+	principalRoleName := fmt.Sprintf("polaris_principal_role_%d", time.Now().UnixNano())
+	catalogRoleName := fmt.Sprintf("polaris_catalog_role_%d", time.Now().UnixNano())
+
+	var principalResp createPrincipalResponse
+	if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principals", map[string]interface{}{
+		"principal": map[string]interface{}{
+			"name":       principalName,
+			"properties": map[string]string{},
+		},
+	}, &principalResp); err != nil {
+		t.Fatalf("Create principal failed: %v", err)
+	}
+	if principalResp.Credentials.ClientID == "" || principalResp.Credentials.ClientSecret == "" {
+		t.Fatalf("Missing principal credentials in response")
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principal-roles", map[string]interface{}{
+		"principalRole": map[string]interface{}{
+			"name":       principalRoleName,
+			"properties": map[string]string{},
+		},
+	}, nil); err != nil {
+		t.Fatalf("Create principal role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPost, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles", url.PathEscape(catalogName)), map[string]interface{}{
+		"catalogRole": map[string]interface{}{
+			"name":       catalogRoleName,
+			"properties": map[string]string{},
+		},
+	}, nil); err != nil {
+		t.Fatalf("Create catalog role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principals/%s/principal-roles", url.PathEscape(principalName)), map[string]interface{}{
+		"principalRole": map[string]interface{}{
+			"name": principalRoleName,
+		},
+	}, nil); err != nil {
+		t.Fatalf("Assign principal role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principal-roles/%s/catalog-roles/%s", url.PathEscape(principalRoleName), url.PathEscape(catalogName)), map[string]interface{}{
+		"catalogRole": map[string]interface{}{
+			"name": catalogRoleName,
+		},
+	}, nil); err != nil {
+		t.Fatalf("Assign catalog role failed: %v", err)
+	}
+
+	if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles/%s/grants", url.PathEscape(catalogName), url.PathEscape(catalogRoleName)), map[string]interface{}{
+		"type":      "catalog",
+		"privilege": "CATALOG_MANAGE_CONTENT",
+	}, nil); err != nil {
+		t.Fatalf("Grant catalog privilege failed: %v", err)
+	}
+
+	userToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), principalResp.Credentials.ClientID, principalResp.Credentials.ClientSecret)
+	if err != nil {
+		t.Fatalf("Polaris user token request failed: %v", err)
+	}
+
+	return polarisSession{
+		catalogName:  catalogName,
+		bucketName:   bucketName,
+		token:        userToken,
+		baseLocation: baseLocation,
+	}
+}
+
+func setupPolarisTable(t *testing.T, ctx context.Context, env *TestEnvironment, session polarisSession) (*polarisTableSetup, func()) {
+	t.Helper()
+
+	catalogClient := newPolarisCatalogClient(env.polarisEndpoint(), polarisRealm, session.token, session.catalogName)
+	namespace := fmt.Sprintf("polaris_ns_%d", time.Now().UnixNano())
+	table := fmt.Sprintf("polaris_table_%d", time.Now().UnixNano())
+
+	if err := catalogClient.CreateNamespace(ctx, namespace); err != nil {
+		t.Fatalf("CreateNamespace failed: %v", err)
+	}
+
+	location := fmt.Sprintf("%s/%s/%s", session.baseLocation, namespace, table)
+	if err := catalogClient.CreateTable(ctx, namespace, table, location); err != nil {
+		t.Fatalf("CreateTable failed: %v", err)
+	}
+
+	loadResp, err := catalogClient.LoadTable(ctx, namespace, table)
+	if err != nil {
+		t.Fatalf("LoadTable failed: %v", err)
+	}
+
+	credsResp, err := catalogClient.LoadCredentials(ctx, namespace, table)
+	if err != nil {
+		t.Fatalf("LoadCredentials failed: %v", err)
+	}
+	if len(credsResp.StorageCredentials) == 0 {
+		t.Fatalf("LoadCredentials returned no storage credentials")
+	}
+
+	credentialSource := &loadTableResponse{
+		Config:             loadResp.Config,
+		StorageCredentials: credsResp.StorageCredentials,
+	}
+
+	dataPrefix := location + "/data"
+	creds, endpoint, region, pathStyle, err := extractS3Credentials(credentialSource, dataPrefix, env.s3Endpoint(), polarisRegion)
+	if err != nil {
+		t.Fatalf("Extract vended credentials failed: %v", err)
+	}
+	dataKeyPrefix, err := s3URIToKeyPrefix(dataPrefix, session.bucketName)
+	if err != nil {
+		t.Fatalf("Invalid data prefix %s: %v", dataPrefix, err)
+	}
+
+	s3Client, err := newS3Client(ctx, endpoint, region, creds, pathStyle)
+	if err != nil {
+		t.Fatalf("Create vended S3 client failed: %v", err)
+	}
+
+	cleanup := func() {
+		cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cleanupCancel()
+		if err := catalogClient.DeleteTable(cleanupCtx, namespace, table); err != nil {
+			t.Logf("DeleteTable failed: %v", err)
+		}
+		if err := catalogClient.DeleteNamespace(cleanupCtx, namespace); err != nil {
+			t.Logf("DeleteNamespace failed: %v", err)
+		}
+	}
+
+	return &polarisTableSetup{
+		namespace:     namespace,
+		table:         table,
+		dataKeyPrefix: dataKeyPrefix,
+		s3Client:      s3Client,
+	}, cleanup
+}
+
+func fetchPolarisToken(ctx context.Context, baseURL, clientID, clientSecret string) (string, error) {
+	form := url.Values{}
+	form.Set("grant_type", "client_credentials")
+	form.Set("client_id", clientID)
+	form.Set("client_secret", clientSecret)
+	form.Set("scope", "PRINCIPAL_ROLE:ALL")
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/catalog/v1/oauth/tokens", strings.NewReader(form.Encode()))
+	if err != nil {
+		return "", fmt.Errorf("create token request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Accept", "application/json")
+
+	client := &http.Client{Timeout: 30 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("token request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, readErr := io.ReadAll(resp.Body)
+		if readErr != nil {
+			return "", fmt.Errorf("token request failed with status %d and reading body: %w", resp.StatusCode, readErr)
+		}
+		return "", fmt.Errorf("token request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+
+	var tokenResp struct {
+		AccessToken string `json:"access_token"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
+		return "", fmt.Errorf("decode token response: %w", err)
+	}
+	if tokenResp.AccessToken == "" {
+		return "", fmt.Errorf("missing access token in response")
+	}
+
+	return tokenResp.AccessToken, nil
+}
+
+func newPolarisCatalogClient(baseURL, realm, token, catalog string) *polarisCatalogClient {
+	return &polarisCatalogClient{
+		http:    newPolarisHTTPClient(baseURL, realm, token),
+		catalog: catalog,
+	}
+}
+
+func (c *polarisCatalogClient) CreateNamespace(ctx context.Context, namespace string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces", url.PathEscape(c.catalog))
+	req := createNamespaceRequest{Namespace: []string{namespace}}
+	return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
+}
+
+func (c *polarisCatalogClient) DeleteNamespace(ctx context.Context, namespace string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s", url.PathEscape(c.catalog), url.PathEscape(namespace))
+	return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
+}
+
+func (c *polarisCatalogClient) CreateTable(ctx context.Context, namespace, table, location string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables", url.PathEscape(c.catalog), url.PathEscape(namespace))
+
+	req := createTableRequest{
+		Name:     table,
+		Location: location,
+		Schema: icebergSchema{
+			Type:     "struct",
+			SchemaID: 0,
+			Fields: []icebergSchemaField{
+				{
+					ID:       1,
+					Name:     "id",
+					Type:     "long",
+					Required: false,
+				},
+			},
+		},
+		PartitionSpec: icebergPartition{
+			SpecID: 0,
+			Fields: []icebergPartitionField{},
+		},
+		SortOrder: icebergSortOrder{
+			OrderID: 0,
+			Fields:  []icebergSortField{},
+		},
+		Properties: map[string]string{
+			"format-version": "2",
+		},
+	}
+
+	return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
+}
+
+func (c *polarisCatalogClient) DeleteTable(ctx context.Context, namespace, table string) error {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
+	return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
+}
+
+func (c *polarisCatalogClient) LoadTable(ctx context.Context, namespace, table string) (*loadTableResponse, error) {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
+	var resp loadTableResponse
+	headers := map[string]string{
+		"X-Iceberg-Access-Delegation": "vended-credentials",
+	}
+	if err := c.http.doJSONWithHeaders(ctx, http.MethodGet, path, nil, &resp, headers); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, table string) (*loadCredentialsResponse, error) {
+	path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s/credentials", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
+	var resp loadCredentialsResponse
+	if err := c.http.doJSON(ctx, http.MethodGet, path, nil, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
+	configMap, err := selectStorageConfig(load, targetPrefix)
+	if err != nil {
+		return aws.Credentials{}, "", "", false, err
+	}
+
+	lookup := func(keys ...string) string {
+		for _, key := range keys {
+			if val, ok := configMap[key]; ok && val != "" {
+				return val
+			}
+		}
+		return ""
+	}
+
+	accessKey := lookup("s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id")
+	secretKey := lookup("s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key")
+	sessionToken := lookup("s3.session-token", "aws.session-token", "s3.sessionToken", "aws.sessionToken", "sessionToken", "session_token")
+	if accessKey == "" || secretKey == "" {
+		if altConfig := findConfigWithKeys(load, targetPrefix); altConfig != nil {
+			configMap = altConfig
+			accessKey = lookup("s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id")
+			secretKey = lookup("s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key")
+			sessionToken = lookup("s3.session-token", "aws.session-token", "s3.sessionToken", "aws.sessionToken", "sessionToken", "session_token")
+		}
+	}
+	if accessKey == "" || secretKey == "" {
+		return aws.Credentials{}, "", "", false, fmt.Errorf("missing access key or secret in storage credentials")
+	}
+
+	endpoint := lookup("s3.endpoint", "s3.endpoint-url", "aws.endpoint")
+	if endpoint == "" {
+		endpoint = fallbackEndpoint
+	}
+	if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
+		endpoint = "http://" + endpoint
+	}
+
+	region := lookup("s3.region", "aws.region")
+	if region == "" {
+		region = fallbackRegion
+	}
+
+	pathStyle := true
+	if value := lookup("s3.path-style-access", "s3.pathStyleAccess", "path-style-access"); value != "" {
+		pathStyle = strings.EqualFold(value, "true")
+	}
+
+	return aws.Credentials{
+		AccessKeyID:     accessKey,
+		SecretAccessKey: secretKey,
+		SessionToken:    sessionToken,
+		Source:          "polaris-vended",
+	}, endpoint, region, pathStyle, nil
+}
+
+func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
+	switch len(load.StorageCredentials) {
+	case 0:
+		if load.Config == nil {
+			return nil, fmt.Errorf("polaris returned no storage credentials or config")
+		}
+		return load.Config, nil
+	case 1:
+		cred := load.StorageCredentials[0]
+		if cred.Config == nil {
+			return nil, fmt.Errorf("storage credential for prefix %s returned nil config", cred.Prefix)
+		}
+		return cred.Config, nil
+	default:
+		if targetPrefix == "" {
+			return nil, fmt.Errorf("multiple storage credentials (%d) returned but no target prefix provided", len(load.StorageCredentials))
+		}
+		normalizedTarget := normalizePrefix(targetPrefix)
+		if normalizedTarget == "" {
+			return nil, fmt.Errorf("target prefix %q normalized to empty string", targetPrefix)
+		}
+		var bestConfig map[string]string
+		bestLen := -1
+		for _, cred := range load.StorageCredentials {
+			if cred.Config == nil {
+				continue
+			}
+			prefix := normalizePrefix(cred.Prefix)
+			if prefix == "" {
+				if bestLen < 0 {
+					bestLen = 0
+					bestConfig = cred.Config
+				}
+				continue
+			}
+			if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
+				if len(prefix) > bestLen {
+					bestLen = len(prefix)
+					bestConfig = cred.Config
+				}
+			}
+		}
+		if bestConfig != nil {
+			return bestConfig, nil
+		}
+		return nil, fmt.Errorf("none of the %d storage credentials matched prefix %s", len(load.StorageCredentials), targetPrefix)
+	}
+}
+
+func normalizePrefix(prefix string) string {
+	p := strings.TrimSpace(prefix)
+	p = strings.TrimSuffix(p, "/")
+	return p
+}
+
+func findConfigWithKeys(load *loadTableResponse, targetPrefix string) map[string]string {
+	normalizedTarget := normalizePrefix(targetPrefix)
+	if normalizedTarget == "" {
+		return nil
+	}
+
+	hasKeys := func(config map[string]string) bool {
+		if config == nil {
+			return false
+		}
+		accessKey := lookupValue(config, "s3.access-key-id", "aws.access-key-id", "s3.accessKeyId", "aws.accessKeyId", "accessKeyId", "access_key_id")
+		secretKey := lookupValue(config, "s3.secret-access-key", "aws.secret-access-key", "s3.secretAccessKey", "aws.secretAccessKey", "secretAccessKey", "secret_access_key")
+		return accessKey != "" && secretKey != ""
+	}
+
+	bestConfig := map[string]string(nil)
+	bestLen := -1
+	for _, cred := range load.StorageCredentials {
+		if !hasKeys(cred.Config) {
+			continue
+		}
+		prefix := normalizePrefix(cred.Prefix)
+		if prefix == "" {
+			if bestLen < 0 {
+				bestLen = 0
+				bestConfig = cred.Config
+			}
+			continue
+		}
+		if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
+			if len(prefix) > bestLen {
+				bestLen = len(prefix)
+				bestConfig = cred.Config
+			}
+		}
+	}
+	if bestConfig != nil {
+		return bestConfig
+	}
+
+	if len(load.StorageCredentials) == 0 && hasKeys(load.Config) {
+		return load.Config
+	}
+	if hasKeys(load.Config) {
+		return load.Config
+	}
+	return nil
+}
+
+func lookupValue(config map[string]string, keys ...string) string {
+	for _, key := range keys {
+		if val, ok := config[key]; ok && val != "" {
+			return val
+		}
+	}
+	return ""
+}
+
+func s3URIToKeyPrefix(uri, bucket string) (string, error) {
+	prefix := "s3://" + bucket + "/"
+	if !strings.HasPrefix(uri, prefix) {
+		return "", fmt.Errorf("uri %q does not match bucket %q", uri, bucket)
+	}
+	keyPrefix := strings.TrimPrefix(uri, prefix)
+	keyPrefix = strings.TrimPrefix(keyPrefix, "/")
+	if keyPrefix == "" {
+		return "", fmt.Errorf("empty key prefix in uri %q", uri)
+	}
+	return keyPrefix, nil
+}