gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: remove ~50k lines of unreachable dead code (#8913)

Browse Source 
* chore: remove unreachable dead code across the codebase

Remove ~50,000 lines of unreachable code identified by static analysis.

Major removals:
- weed/filer/redis_lua: entire unused Redis Lua filer store implementation
- weed/wdclient/net2, resource_pool: unused connection/resource pool packages
- weed/plugin/worker/lifecycle: unused lifecycle plugin worker
- weed/s3api: unused S3 policy templates, presigned URL IAM, streaming copy,
  multipart IAM, key rotation, and various SSE helper functions
- weed/mq/kafka: unused partition mapping, compression, schema, and protocol functions
- weed/mq/offset: unused SQL storage and migration code
- weed/worker: unused registry, task, and monitoring functions
- weed/query: unused SQL engine, parquet scanner, and type functions
- weed/shell: unused EC proportional rebalance functions
- weed/storage/erasure_coding/distribution: unused distribution analysis functions
- Individual unreachable functions removed from 150+ files across admin,
  credential, filer, iam, kms, mount, mq, operation, pb, s3api, server,
  shell, storage, topology, and util packages

* fix(s3): reset shared memory store in IAM test to prevent flaky failure

TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey was flaky because
the MemoryStore credential backend is a singleton registered via init().
Earlier tests that create anonymous identities pollute the shared store,
causing LookupAnonymous() to unexpectedly return true.

Fix by calling Reset() on the memory store before the test runs.

* style: run gofmt on changed files

* fix: restore KMS functions used by integration tests

* fix(plugin): prevent panic on send to closed worker session channel

The Plugin.sendToWorker method could panic with "send on closed channel"
when a worker disconnected while a message was being sent. The race was
between streamSession.close() closing the outgoing channel and sendToWorker
writing to it concurrently.

Add a done channel to streamSession that is closed before the outgoing
channel, and check it in sendToWorker's select to safely detect closed
sessions without panicking.
This commit is contained in:
Chris Lu
2026-04-03 16:04:27 -07:00
committed by GitHub
parent 8fad85aed7
commit 995dfc4d5d
264 changed files with 62 additions and 46027 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
weed/s3api/s3tables/filer_ops.go
View File

@@ -50,46 +50,6 @@ func (h *S3TablesHandler) ensureDirectory(ctx context.Context, client filer_pb.S
return err
}
// upsertFile creates or updates a small file with the given content
func (h *S3TablesHandler) upsertFile(ctx context.Context, client filer_pb.SeaweedFilerClient, path string, data []byte) error {
dir, name := splitPath(path)
now := time.Now().Unix()
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
if !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
return filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
Directory: dir,
Entry: &filer_pb.Entry{
Name: name,
Content: data,
Attributes: &filer_pb.FuseAttributes{
Mtime: now,
Crtime: now,
FileMode: uint32(0644),
FileSize: uint64(len(data)),
},
},
})
}
entry := resp.Entry
if entry.Attributes == nil {
entry.Attributes = &filer_pb.FuseAttributes{}
}
entry.Attributes.Mtime = now
entry.Attributes.FileSize = uint64(len(data))
entry.Content = data
return filer_pb.UpdateEntry(ctx, client, &filer_pb.UpdateEntryRequest{
Directory: dir,
Entry: entry,
})
}
// deleteEntryIfExists removes an entry if it exists, ignoring missing errors
func (h *S3TablesHandler) deleteEntryIfExists(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
dir, name := splitPath(path)

132
weed/s3api/s3tables/iceberg_layout.go
View File

@@ -1,14 +1,9 @@
package s3tables
import (
"context"
"encoding/json"
"errors"
pathpkg "path"
"regexp"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
// Iceberg file layout validation
@@ -307,130 +302,3 @@ func (v *TableBucketFileValidator) ValidateTableBucketUpload(fullPath string) er
return v.layoutValidator.ValidateFilePath(tableRelativePath)
}
// IsTableBucketPath checks if a path is under the table buckets directory
func IsTableBucketPath(fullPath string) bool {
return strings.HasPrefix(fullPath, TablesPath+"/")
}
// GetTableInfoFromPath extracts bucket, namespace, and table names from a table bucket path
// Returns empty strings if the path doesn't contain enough components
func GetTableInfoFromPath(fullPath string) (bucket, namespace, table string) {
if !strings.HasPrefix(fullPath, TablesPath+"/") {
return "", "", ""
}
relativePath := strings.TrimPrefix(fullPath, TablesPath+"/")
parts := strings.SplitN(relativePath, "/", 4)
if len(parts) >= 1 {
bucket = parts[0]
}
if len(parts) >= 2 {
namespace = parts[1]
}
if len(parts) >= 3 {
table = parts[2]
}
return
}
// ValidateTableBucketUploadWithClient validates upload and checks that the table exists and is ICEBERG format
func (v *TableBucketFileValidator) ValidateTableBucketUploadWithClient(
ctx context.Context,
client filer_pb.SeaweedFilerClient,
fullPath string,
) error {
// If not a table bucket path, nothing more to check
if !IsTableBucketPath(fullPath) {
return nil
}
// Get table info and verify it exists
bucket, namespace, table := GetTableInfoFromPath(fullPath)
if bucket == "" || namespace == "" || table == "" {
return nil // Not deep enough to need validation
}
if strings.HasPrefix(bucket, ".") {
return nil
}
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: TablesPath,
Name: bucket,
})
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return nil
}
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "failed to verify table bucket: " + err.Error(),
}
}
if resp == nil || !IsTableBucketEntry(resp.Entry) {
return nil
}
// Now check basic layout once we know this is a table bucket path.
if err := v.ValidateTableBucketUpload(fullPath); err != nil {
return err
}
// Verify the table exists and has ICEBERG format by checking its metadata
tablePath := GetTablePath(bucket, namespace, table)
dir, name := splitPath(tablePath)
resp, err = filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if err != nil {
// Distinguish between "not found" and other errors
if errors.Is(err, filer_pb.ErrNotFound) {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table does not exist",
}
}
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "failed to verify table existence: " + err.Error(),
}
}
// Check if table has metadata indicating ICEBERG format
if resp.Entry == nil || resp.Entry.Extended == nil {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table is not a valid ICEBERG table (missing metadata)",
}
}
metadataBytes, ok := resp.Entry.Extended[ExtendedKeyMetadata]
if !ok {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table is not in ICEBERG format (missing format metadata)",
}
}
var metadata tableMetadataInternal
if err := json.Unmarshal(metadataBytes, &metadata); err != nil {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "failed to parse table metadata: " + err.Error(),
}
}
const TableFormatIceberg = "ICEBERG"
if metadata.Format != TableFormatIceberg {
return &IcebergLayoutError{
Code: ErrCodeInvalidIcebergLayout,
Message: "table is not in " + TableFormatIceberg + " format",
}
}
return nil
}

186
weed/s3api/s3tables/iceberg_layout_test.go
View File

@@ -1,186 +0,0 @@
package s3tables
import (
"testing"
)
func TestIcebergLayoutValidator_ValidateFilePath(t *testing.T) {
v := NewIcebergLayoutValidator()
tests := []struct {
name string
path string
wantErr bool
}{
// Valid metadata files
{"valid metadata v1", "metadata/v1.metadata.json", false},
{"valid metadata v123", "metadata/v123.metadata.json", false},
{"valid snapshot manifest", "metadata/snap-123-1-abc12345-1234-5678-9abc-def012345678.avro", false},
{"valid manifest file", "metadata/abc12345-1234-5678-9abc-def012345678-m0.avro", false},
{"valid general manifest", "metadata/abc12345-1234-5678-9abc-def012345678.avro", false},
{"valid version hint", "metadata/version-hint.text", false},
{"valid uuid metadata", "metadata/abc12345-1234-5678-9abc-def012345678.metadata.json", false},
{"valid trino stats", "metadata/20260208_212535_00007_bn4hb-d3599c32-1709-4b94-b6b2-1957b6d6db04.stats", false},
// Valid data files
{"valid parquet file", "data/file.parquet", false},
{"valid orc file", "data/file.orc", false},
{"valid avro data file", "data/file.avro", false},
{"valid parquet with path", "data/00000-0-abc12345.parquet", false},
// Valid partitioned data
{"valid partitioned parquet", "data/year=2024/file.parquet", false},
{"valid multi-partition", "data/year=2024/month=01/file.parquet", false},
{"valid bucket subdirectory", "data/bucket0/file.parquet", false},
// Directories only
{"metadata directory bare", "metadata", true},
{"data directory bare", "data", true},
{"metadata directory with slash", "metadata/", false},
{"data directory with slash", "data/", false},
// Invalid paths
{"empty path", "", true},
{"invalid top dir", "invalid/file.parquet", true},
{"root file", "file.parquet", true},
{"invalid metadata file", "metadata/random.txt", true},
{"nested metadata directory", "metadata/nested/v1.metadata.json", true},
{"nested metadata directory no file", "metadata/nested/", true},
{"metadata subdir no slash", "metadata/nested", true},
{"invalid data file", "data/file.csv", true},
{"invalid data file json", "data/file.json", true},
// Partition/subdirectory without trailing slashes
{"partition directory no slash", "data/year=2024", false},
{"data subdirectory no slash", "data/my_subdir", false},
{"multi-level partition", "data/event_date=2025-01-01/hour=00/file.parquet", false},
{"multi-level partition directory", "data/event_date=2025-01-01/hour=00/", false},
{"multi-level partition directory no slash", "data/event_date=2025-01-01/hour=00", false},
// Double slashes
{"data double slash", "data//file.parquet", true},
{"data redundant slash", "data/year=2024//file.parquet", true},
{"metadata redundant slash", "metadata//v1.metadata.json", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := v.ValidateFilePath(tt.path)
if (err != nil) != tt.wantErr {
t.Errorf("ValidateFilePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
}
})
}
}
func TestIcebergLayoutValidator_PartitionPaths(t *testing.T) {
v := NewIcebergLayoutValidator()
validPaths := []string{
"data/year=2024/file.parquet",
"data/date=2024-01-15/file.parquet",
"data/category=electronics/file.parquet",
"data/user_id=12345/file.parquet",
"data/region=us-east-1/file.parquet",
"data/year=2024/month=01/day=15/file.parquet",
}
for _, path := range validPaths {
if err := v.ValidateFilePath(path); err != nil {
t.Errorf("ValidateFilePath(%q) should be valid, got error: %v", path, err)
}
}
}
func TestTableBucketFileValidator_ValidateTableBucketUpload(t *testing.T) {
v := NewTableBucketFileValidator()
tests := []struct {
name string
path string
wantErr bool
}{
// Non-table bucket paths should pass (no validation)
{"regular bucket path", "/buckets/mybucket/file.txt", false},
{"filer path", "/home/user/file.txt", false},
// Table bucket structure paths (creating directories)
{"table bucket root", "/buckets/mybucket", false},
{"namespace dir", "/buckets/mybucket/myns", false},
{"table dir", "/buckets/mybucket/myns/mytable", false},
{"table dir trailing slash", "/buckets/mybucket/myns/mytable/", false},
// Valid table bucket file uploads
{"valid parquet upload", "/buckets/mybucket/myns/mytable/data/file.parquet", false},
{"valid metadata upload", "/buckets/mybucket/myns/mytable/metadata/v1.metadata.json", false},
{"valid trino stats upload", "/buckets/mybucket/myns/mytable/metadata/20260208_212535_00007_bn4hb-d3599c32-1709-4b94-b6b2-1957b6d6db04.stats", false},
{"valid partitioned data", "/buckets/mybucket/myns/mytable/data/year=2024/file.parquet", false},
// Invalid table bucket file uploads
{"invalid file type", "/buckets/mybucket/myns/mytable/data/file.csv", true},
{"invalid top-level dir", "/buckets/mybucket/myns/mytable/invalid/file.parquet", true},
{"root file in table", "/buckets/mybucket/myns/mytable/file.parquet", true},
// Empty segment cases
{"empty bucket", "/buckets//myns/mytable/data/file.parquet", true},
{"empty namespace", "/buckets/mybucket//mytable/data/file.parquet", true},
{"empty table", "/buckets/mybucket/myns//data/file.parquet", true},
{"empty bucket dir", "/buckets//", true},
{"empty namespace dir", "/buckets/mybucket//", true},
{"table double slash bypass", "/buckets/mybucket/myns/mytable//data/file.parquet", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := v.ValidateTableBucketUpload(tt.path)
if (err != nil) != tt.wantErr {
t.Errorf("ValidateTableBucketUpload(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
}
})
}
}
func TestIsTableBucketPath(t *testing.T) {
tests := []struct {
path string
want bool
}{
{"/buckets/mybucket", true},
{"/buckets/mybucket/ns/table/data/file.parquet", true},
{"/home/user/file.txt", false},
{"buckets/mybucket", false}, // missing leading slash
}
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
if got := IsTableBucketPath(tt.path); got != tt.want {
t.Errorf("IsTableBucketPath(%q) = %v, want %v", tt.path, got, tt.want)
}
})
}
}
func TestGetTableInfoFromPath(t *testing.T) {
tests := []struct {
path string
wantBucket string
wantNamespace string
wantTable string
}{
{"/buckets/mybucket/myns/mytable/data/file.parquet", "mybucket", "myns", "mytable"},
{"/buckets/mybucket/myns/mytable", "mybucket", "myns", "mytable"},
{"/buckets/mybucket/myns", "mybucket", "myns", ""},
{"/buckets/mybucket", "mybucket", "", ""},
{"/home/user/file.txt", "", "", ""},
}
for _, tt := range tests {
t.Run(tt.path, func(t *testing.T) {
bucket, namespace, table := GetTableInfoFromPath(tt.path)
if bucket != tt.wantBucket || namespace != tt.wantNamespace || table != tt.wantTable {
t.Errorf("GetTableInfoFromPath(%q) = (%q, %q, %q), want (%q, %q, %q)",
tt.path, bucket, namespace, table, tt.wantBucket, tt.wantNamespace, tt.wantTable)
}
})
}
}

118
weed/s3api/s3tables/permissions.go
View File

@@ -90,17 +90,6 @@ type PolicyContext struct {
DefaultAllow bool
}
// CheckPermissionWithResource checks if a principal has permission to perform an operation on a specific resource
func CheckPermissionWithResource(operation, principal, owner, resourcePolicy, resourceARN string) bool {
return CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN, nil)
}
// CheckPermission checks if a principal has permission to perform an operation
// (without resource-specific validation - for backward compatibility)
func CheckPermission(operation, principal, owner, resourcePolicy string) bool {
return CheckPermissionWithContext(operation, principal, owner, resourcePolicy, "", nil)
}
// CheckPermissionWithContext checks permission with optional resource and condition context.
func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
// Deny access if identities are empty
@@ -415,113 +404,6 @@ func matchesResourcePattern(pattern, resourceARN string) bool {
return wildcard.MatchesWildcard(pattern, resourceARN)
}
// Helper functions for specific permissions
// CanCreateTableBucket checks if principal can create table buckets
func CanCreateTableBucket(principal, owner, resourcePolicy string) bool {
return CheckPermission("CreateTableBucket", principal, owner, resourcePolicy)
}
// CanGetTableBucket checks if principal can get table bucket details
func CanGetTableBucket(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTableBucket", principal, owner, resourcePolicy)
}
// CanListTableBuckets checks if principal can list table buckets
func CanListTableBuckets(principal, owner, resourcePolicy string) bool {
return CheckPermission("ListTableBuckets", principal, owner, resourcePolicy)
}
// CanDeleteTableBucket checks if principal can delete table buckets
func CanDeleteTableBucket(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTableBucket", principal, owner, resourcePolicy)
}
// CanPutTableBucketPolicy checks if principal can put table bucket policies
func CanPutTableBucketPolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("PutTableBucketPolicy", principal, owner, resourcePolicy)
}
// CanGetTableBucketPolicy checks if principal can get table bucket policies
func CanGetTableBucketPolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTableBucketPolicy", principal, owner, resourcePolicy)
}
// CanDeleteTableBucketPolicy checks if principal can delete table bucket policies
func CanDeleteTableBucketPolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTableBucketPolicy", principal, owner, resourcePolicy)
}
// CanCreateNamespace checks if principal can create namespaces
func CanCreateNamespace(principal, owner, resourcePolicy string) bool {
return CheckPermission("CreateNamespace", principal, owner, resourcePolicy)
}
// CanGetNamespace checks if principal can get namespace details
func CanGetNamespace(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetNamespace", principal, owner, resourcePolicy)
}
// CanListNamespaces checks if principal can list namespaces
func CanListNamespaces(principal, owner, resourcePolicy string) bool {
return CheckPermission("ListNamespaces", principal, owner, resourcePolicy)
}
// CanDeleteNamespace checks if principal can delete namespaces
func CanDeleteNamespace(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteNamespace", principal, owner, resourcePolicy)
}
// CanCreateTable checks if principal can create tables
func CanCreateTable(principal, owner, resourcePolicy string) bool {
return CheckPermission("CreateTable", principal, owner, resourcePolicy)
}
// CanGetTable checks if principal can get table details
func CanGetTable(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTable", principal, owner, resourcePolicy)
}
// CanListTables checks if principal can list tables
func CanListTables(principal, owner, resourcePolicy string) bool {
return CheckPermission("ListTables", principal, owner, resourcePolicy)
}
// CanDeleteTable checks if principal can delete tables
func CanDeleteTable(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTable", principal, owner, resourcePolicy)
}
// CanPutTablePolicy checks if principal can put table policies
func CanPutTablePolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("PutTablePolicy", principal, owner, resourcePolicy)
}
// CanGetTablePolicy checks if principal can get table policies
func CanGetTablePolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("GetTablePolicy", principal, owner, resourcePolicy)
}
// CanDeleteTablePolicy checks if principal can delete table policies
func CanDeleteTablePolicy(principal, owner, resourcePolicy string) bool {
return CheckPermission("DeleteTablePolicy", principal, owner, resourcePolicy)
}
// CanTagResource checks if principal can tag a resource
func CanTagResource(principal, owner, resourcePolicy string) bool {
return CheckPermission("TagResource", principal, owner, resourcePolicy)
}
// CanUntagResource checks if principal can untag a resource
func CanUntagResource(principal, owner, resourcePolicy string) bool {
return CheckPermission("UntagResource", principal, owner, resourcePolicy)
}
// CanManageTags checks if principal can manage tags (tag or untag)
func CanManageTags(principal, owner, resourcePolicy string) bool {
return CanTagResource(principal, owner, resourcePolicy) || CanUntagResource(principal, owner, resourcePolicy)
}
// AuthError represents an authorization error
type AuthError struct {
Operation string

10
weed/s3api/s3tables/utils.go
View File

@@ -200,11 +200,6 @@ func validateBucketName(name string) error {
return nil
}
// ValidateBucketName validates bucket name and returns an error if invalid.
func ValidateBucketName(name string) error {
return validateBucketName(name)
}
// BuildBucketARN builds a bucket ARN with the provided region and account ID.
// If region is empty, the ARN will omit the region field.
func BuildBucketARN(region, accountID, bucketName string) (string, error) {
@@ -367,11 +362,6 @@ func validateNamespace(namespace []string) (string, error) {
return flattenNamespace(parts), nil
}
// ValidateNamespace is a wrapper to validate namespace for other packages.
func ValidateNamespace(namespace []string) (string, error) {
return validateNamespace(namespace)
}
// ParseNamespace parses a namespace string into namespace parts.
func ParseNamespace(namespace string) ([]string, error) {
return normalizeNamespace([]string{namespace})