gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/seaweedfs/seaweedfs

Browse Source
This commit is contained in:
Chris Lu
2026-02-08 20:05:24 -08:00
parent 3bb9493a5b be6b5db65a
commit bbcb8b7590
20 changed files with 1114 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
weed/admin/view/app/maintenance_workers.templ
View File

@@ -118,7 +118,7 @@ templ MaintenanceWorkers(data *dash.MaintenanceWorkersData) {
<p class="text-muted">No maintenance workers are currently registered.</p>
<div class="alert alert-info mt-3">
<strong>Tip:</strong> To start a worker, run:
<br><code>weed worker -admin=&lt;admin_server&gt; -capabilities=vacuum,ec,replication</code>
<br><code>weed worker -admin=&lt;admin_server&gt; -capabilities=vacuum,ec,balance</code>
</div>
</div>
} else {
@@ -340,4 +340,4 @@ templ MaintenanceWorkers(data *dash.MaintenanceWorkersData) {
}
}
</script>
}
}

2
weed/admin/view/app/maintenance_workers_templ.go
View File

@@ -105,7 +105,7 @@ func MaintenanceWorkers(data *dash.MaintenanceWorkersData) templ.Component {
return templ_7745c5c3_Err
}
if len(data.Workers) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<div class=\"text-center py-4\"><i class=\"fas fa-users fa-3x text-gray-300 mb-3\"></i><h5 class=\"text-gray-600\">No Workers Found</h5><p class=\"text-muted\">No maintenance workers are currently registered.</p><div class=\"alert alert-info mt-3\"><strong>Tip:</strong> To start a worker, run:<br><code>weed worker -admin=&lt;admin_server&gt; -capabilities=vacuum,ec,replication</code></div></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<div class=\"text-center py-4\"><i class=\"fas fa-users fa-3x text-gray-300 mb-3\"></i><h5 class=\"text-gray-600\">No Workers Found</h5><p class=\"text-muted\">No maintenance workers are currently registered.</p><div class=\"alert alert-info mt-3\"><strong>Tip:</strong> To start a worker, run:<br><code>weed worker -admin=&lt;admin_server&gt; -capabilities=vacuum,ec,balance</code></div></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}

6
weed/filer/filer.go
View File

@@ -276,8 +276,10 @@ func (f *Filer) ensureParentDirectoryEntry(ctx context.Context, entry *Entry, di
// dirParts[0] == "" and dirParts[1] == "buckets"
isUnderBuckets := len(dirParts) >= 3 && dirParts[1] == "buckets"
if isUnderBuckets {
if err := s3bucket.VerifyS3BucketName(dirParts[2]); err != nil {
return fmt.Errorf("invalid bucket name %s: %v", dirParts[2], err)
if !strings.HasPrefix(dirParts[2], ".") {
if err := s3bucket.VerifyS3BucketName(dirParts[2]); err != nil {
return fmt.Errorf("invalid bucket name %s: %v", dirParts[2], err)
}
}
}

3
weed/s3api/bucket_paths.go
View File

@@ -89,9 +89,6 @@ func (s3a *S3ApiServer) bucketDir(bucket string) string {
if tablePath, ok := s3a.tableLocationDir(bucket); ok {
return tablePath
}
if s3a.isTableBucket(bucket) {
return s3tables.GetTableObjectBucketPath(bucket)
}
return path.Join(s3a.bucketRoot(bucket), bucket)
}

153
weed/s3api/iceberg/iceberg.go
View File

@@ -217,13 +217,14 @@ func (s *Server) saveMetadataFile(ctx context.Context, bucketName, tablePath, me
return nil
}
bucketDir := path.Join(bucketsPath, bucketName)
// 1. Ensure bucket directory exists: <bucketsPath>/<bucket>
if err := ensureDir(bucketsPath, bucketName, "bucket directory"); err != nil {
return err
}
// 2. Ensure table path exists: <bucketsPath>/<bucket>/<tablePath>
tableDir := path.Join(bucketsPath, bucketName)
// 2. Ensure table path exists under the bucket directory
tableDir := bucketDir
if tablePath != "" {
segments := strings.Split(tablePath, "/")
for _, segment := range segments {
@@ -354,6 +355,9 @@ func getBucketFromPrefix(r *http.Request) string {
if prefix := vars["prefix"]; prefix != "" {
return prefix
}
if bucket := os.Getenv("S3TABLES_DEFAULT_BUCKET"); bucket != "" {
return bucket
}
// Default bucket if no prefix - use "warehouse" for Iceberg
return "warehouse"
}
@@ -680,24 +684,32 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
// Generate UUID for the new table
tableUUID := uuid.New()
location := strings.TrimSuffix(req.Location, "/")
tablePath := path.Join(encodeNamespace(namespace), req.Name)
storageBucket := bucketName
tableLocationBucket := ""
if location != "" {
location := strings.TrimSuffix(req.Location, "/")
if location == "" {
if req.Properties != nil {
if warehouse := strings.TrimSuffix(req.Properties["warehouse"], "/"); warehouse != "" {
location = fmt.Sprintf("%s/%s", warehouse, tablePath)
}
}
if location == "" {
if warehouse := strings.TrimSuffix(os.Getenv("ICEBERG_WAREHOUSE"), "/"); warehouse != "" {
location = fmt.Sprintf("%s/%s", warehouse, tablePath)
}
}
if location == "" {
location = fmt.Sprintf("s3://%s/%s", bucketName, tablePath)
}
} else {
parsedBucket, parsedPath, err := parseS3Location(location)
if err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid table location: "+err.Error())
return
}
if strings.HasSuffix(parsedBucket, "--table-s3") && parsedPath == "" {
tableLocationBucket = parsedBucket
if parsedPath == "" {
location = fmt.Sprintf("s3://%s/%s", parsedBucket, tablePath)
}
}
if tableLocationBucket == "" {
tableLocationBucket = fmt.Sprintf("%s--table-s3", tableUUID.String())
}
location = fmt.Sprintf("s3://%s", tableLocationBucket)
// Build proper Iceberg table metadata using iceberg-go types
metadata := newTableMetadata(tableUUID, location, req.Schema, req.PartitionSpec, req.WriteOrder, req.Properties)
@@ -713,15 +725,21 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
return
}
// 1. Save metadata file to filer
tableName := req.Name
metadataFileName := "v1.metadata.json" // Initial version is always 1
if err := s.saveMetadataFile(r.Context(), storageBucket, tablePath, metadataFileName, metadataBytes); err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save metadata file: "+err.Error())
return
}
metadataLocation := fmt.Sprintf("%s/metadata/%s", location, metadataFileName)
if !req.StageCreate {
// Save metadata file to filer for immediate table creation.
metadataBucket, metadataPath, err := parseS3Location(location)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Invalid table location: "+err.Error())
return
}
if err := s.saveMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName, metadataBytes); err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save metadata file: "+err.Error())
return
}
}
// Use S3 Tables manager to create table
createReq := &s3tables.CreateTableRequest{
@@ -746,8 +764,42 @@ func (s *Server) handleCreateTable(w http.ResponseWriter, r *http.Request) {
})
if err != nil {
if tableErr, ok := err.(*s3tables.S3TablesError); ok && tableErr.Type == s3tables.ErrCodeTableAlreadyExists {
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
var getResp s3tables.GetTableResponse
getErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
})
if getErr != nil {
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
return
}
result := buildLoadTableResult(getResp, bucketName, namespace, tableName)
writeJSON(w, http.StatusOK, result)
return
}
if strings.Contains(err.Error(), "already exists") {
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
Namespace: namespace,
Name: tableName,
}
var getResp s3tables.GetTableResponse
getErr := s.filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
mgrClient := s3tables.NewManagerClient(client)
return s.tablesManager.Execute(r.Context(), mgrClient, "GetTable", getReq, &getResp, identityName)
})
if getErr != nil {
writeError(w, http.StatusConflict, "AlreadyExistsException", err.Error())
return
}
result := buildLoadTableResult(getResp, bucketName, namespace, tableName)
writeJSON(w, http.StatusOK, result)
return
}
glog.V(1).Infof("Iceberg: CreateTable error: %v", err)
@@ -809,7 +861,11 @@ func (s *Server) handleLoadTable(w http.ResponseWriter, r *http.Request) {
return
}
// Build table metadata using iceberg-go types
result := buildLoadTableResult(getResp, bucketName, namespace, tableName)
writeJSON(w, http.StatusOK, result)
}
func buildLoadTableResult(getResp s3tables.GetTableResponse, bucketName string, namespace []string, tableName string) LoadTableResult {
location := tableLocationFromMetadataLocation(getResp.MetadataLocation)
if location == "" {
location = fmt.Sprintf("s3://%s/%s/%s", bucketName, encodeNamespace(namespace), tableName)
@@ -840,12 +896,11 @@ func (s *Server) handleLoadTable(w http.ResponseWriter, r *http.Request) {
metadata = newTableMetadata(tableUUID, location, nil, nil, nil, nil)
}
result := LoadTableResult{
return LoadTableResult{
MetadataLocation: getResp.MetadataLocation,
Metadata: metadata,
Config: make(iceberg.Properties),
}
writeJSON(w, http.StatusOK, result)
}
// handleTableExists checks if a table exists.
@@ -943,13 +998,53 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
// Extract identity from context
identityName := s3_constants.GetIdentityNameFromContext(r)
// Parse the commit request
var req CommitTableRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
// Parse the commit request, skipping update actions not supported by iceberg-go.
var raw struct {
Identifier *TableIdentifier `json:"identifier,omitempty"`
Requirements json.RawMessage `json:"requirements"`
Updates []json.RawMessage `json:"updates"`
}
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid request body: "+err.Error())
return
}
var req CommitTableRequest
req.Identifier = raw.Identifier
if len(raw.Requirements) > 0 {
if err := json.Unmarshal(raw.Requirements, &req.Requirements); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid requirements: "+err.Error())
return
}
}
if len(raw.Updates) > 0 {
filtered := make([]json.RawMessage, 0, len(raw.Updates))
for _, update := range raw.Updates {
var action struct {
Action string `json:"action"`
}
if err := json.Unmarshal(update, &action); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid update: "+err.Error())
return
}
if action.Action == "set-statistics" {
continue
}
filtered = append(filtered, update)
}
if len(filtered) > 0 {
updatesBytes, err := json.Marshal(filtered)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to parse updates: "+err.Error())
return
}
if err := json.Unmarshal(updatesBytes, &req.Updates); err != nil {
writeError(w, http.StatusBadRequest, "BadRequestException", "Invalid updates: "+err.Error())
return
}
}
}
// First, load current table metadata
getReq := &s3tables.GetTableRequest{
TableBucketARN: bucketARN,
@@ -1049,8 +1144,12 @@ func (s *Server) handleUpdateTable(w http.ResponseWriter, r *http.Request) {
}
// 1. Save metadata file to filer
tablePath := path.Join(encodeNamespace(namespace), tableName)
if err := s.saveMetadataFile(r.Context(), bucketName, tablePath, metadataFileName, metadataBytes); err != nil {
metadataBucket, metadataPath, err := parseS3Location(location)
if err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Invalid table location: "+err.Error())
return
}
if err := s.saveMetadataFile(r.Context(), metadataBucket, metadataPath, metadataFileName, metadataBytes); err != nil {
writeError(w, http.StatusInternalServerError, "InternalServerError", "Failed to save metadata file: "+err.Error())
return
}

1
weed/s3api/s3api_object_handlers_list.go
View File

@@ -524,7 +524,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
stream, listErr := client.ListEntries(ctx, request)
if listErr != nil {
if errors.Is(listErr, filer_pb.ErrNotFound) {
err = filer_pb.ErrNotFound
return
}
err = fmt.Errorf("list entries %+v: %w", request, listErr)

4
weed/s3api/s3api_server.go
View File

@@ -434,8 +434,8 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
s3a.registerS3TablesRoutes(apiRouter)
// Readiness Probe
apiRouter.Methods(http.MethodGet).Path("/status").HandlerFunc(s3a.StatusHandler)
apiRouter.Methods(http.MethodGet).Path("/healthz").HandlerFunc(s3a.StatusHandler)
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/status").HandlerFunc(s3a.StatusHandler)
apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(s3a.StatusHandler)
// Object path pattern with (?s) flag to match newlines in object keys
const objectPath = "/{object:(?s).+}"

8
weed/s3api/s3tables/handler_bucket_create.go
View File

@@ -105,14 +105,6 @@ func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http
}
}
// Ensure object root directory exists for table bucket S3 operations
if err := h.ensureDirectory(r.Context(), client, GetTableObjectRootDir()); err != nil {
return fmt.Errorf("failed to create table object root directory: %w", err)
}
if err := h.ensureDirectory(r.Context(), client, GetTableObjectBucketPath(req.Name)); err != nil {
return fmt.Errorf("failed to create table object bucket directory: %w", err)
}
// Create bucket directory
if err := h.createDirectory(r.Context(), client, bucketPath); err != nil {
return err

32
weed/s3api/s3tables/handler_namespace.go
View File

@@ -50,12 +50,38 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
var bucketMetadata tableBucketMetadata
var bucketPolicy string
var bucketTags map[string]string
ownerAccountID := h.getAccountID(r)
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
if err != nil {
return err
}
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
if errors.Is(err, ErrAttributeNotFound) {
dir, name := splitPath(bucketPath)
entryResp, lookupErr := filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
Directory: dir,
Name: name,
})
if lookupErr != nil {
return lookupErr
}
if entryResp.Entry == nil || !IsTableBucketEntry(entryResp.Entry) {
return filer_pb.ErrNotFound
}
bucketMetadata = tableBucketMetadata{
Name: bucketName,
CreatedAt: time.Now(),
OwnerAccountID: ownerAccountID,
}
metadataBytes, err := json.Marshal(&bucketMetadata)
if err != nil {
return fmt.Errorf("failed to marshal bucket metadata: %w", err)
}
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
return err
}
} else {
return err
}
} else if err := json.Unmarshal(data, &bucketMetadata); err != nil {
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
}

26
weed/s3api/s3tables/handler_table.go
View File

@@ -164,14 +164,26 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
tablePath := GetTablePath(bucketName, namespaceName, tableName)
// Check if table already exists
var existingMetadata tableMetadataInternal
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
return err
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
if err != nil {
return err
}
if unmarshalErr := json.Unmarshal(data, &existingMetadata); unmarshalErr != nil {
return fmt.Errorf("failed to parse existing table metadata: %w", unmarshalErr)
}
return nil
})
if err == nil {
h.writeError(w, http.StatusConflict, ErrCodeTableAlreadyExists, fmt.Sprintf("table %s already exists", tableName))
return fmt.Errorf("table already exists")
tableARN := h.generateTableARN(existingMetadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
h.writeJSON(w, http.StatusOK, &CreateTableResponse{
TableARN: tableARN,
VersionToken: existingMetadata.VersionToken,
MetadataLocation: existingMetadata.MetadataLocation,
})
return nil
} else if !errors.Is(err, filer_pb.ErrNotFound) && !errors.Is(err, ErrAttributeNotFound) {
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
return err
@@ -201,14 +213,14 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
}
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// Create table directory
if err := h.createDirectory(r.Context(), client, tablePath); err != nil {
// Ensure table directory exists (may already be created by object storage clients)
if err := h.ensureDirectory(r.Context(), client, tablePath); err != nil {
return err
}
// Create data subdirectory for Iceberg files
dataPath := tablePath + "/data"
if err := h.createDirectory(r.Context(), client, dataPath); err != nil {
if err := h.ensureDirectory(r.Context(), client, dataPath); err != nil {
return err
}

2
weed/storage/volume_read.go
View File

@@ -265,7 +265,7 @@ func ScanVolumeFileFrom(version needle.Version, datBackend backend.BackendStorag
if err == io.EOF {
return nil
}
return fmt.Errorf("cannot read needle header at offset %d: %v", offset, err)
return fmt.Errorf("cannot read needle header at offset %d: %w", offset, err)
}
glog.V(4).Infof("new entry needle size:%d rest:%d", n.Size, rest)
}