gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add error list each entry func (#7485)

Browse Source 
* added error return in type ListEachEntryFunc

* return error if errClose

* fix fmt.Errorf

* fix return errClose

* use %w fmt.Errorf

* added entry in messege error

* add callbackErr in ListDirectoryEntries

* fix error

* add log

* clear err when the scanner stops on io.EOF, so returning err doesn’t surface EOF as a failure.

* more info in error

* add ctx to logs, error handling

* fix return eachEntryFunc

* fix

* fix log

* fix return

* fix foundationdb test s

* fix eachEntryFunc

* fix return resEachEntryFuncErr

* Update weed/filer/filer.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* Update weed/filer/elastic/v7/elastic_store.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* Update weed/filer/hbase/hbase_store.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* Update weed/filer/foundationdb/foundationdb_store.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* Update weed/filer/ydb/ydb_store.go

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* fix

* add scanErr

---------

Co-authored-by: Roman Tamarov <r.tamarov@kryptonite.ru>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: chrislu <chris.lu@gmail.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
tam-i13
2025-11-26 06:35:19 +03:00
committed by GitHub
parent 76f1a23fec
commit b669607fcd
34 changed files with 350 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
weed/filer/foundationdb/foundationdb_store.go
View File

@@ -318,12 +318,12 @@ func (store *FoundationDBStore) deleteFolderChildrenInBatches(ctx context.Contex
var subDirectories []util.FullPath
// List entries - we'll process BATCH_SIZE at a time
_, err := store.ListDirectoryEntries(ctxNoTxn, fullpath, "", true, int64(BATCH_SIZE), func(entry *filer.Entry) bool {
_, err := store.ListDirectoryEntries(ctxNoTxn, fullpath, "", true, int64(BATCH_SIZE), func(entry *filer.Entry) (bool, error) {
entriesToDelete = append(entriesToDelete, entry.FullPath)
if entry.IsDirectory() {
subDirectories = append(subDirectories, entry.FullPath)
}
return true
return true, nil
})
if err != nil {
@@ -474,9 +474,15 @@ func (store *FoundationDBStore) ListDirectoryPrefixedEntries(ctx context.Context
continue
}
if !eachEntryFunc(entry) {
resEachEntryFunc, resEachEntryFuncErr := eachEntryFunc(entry)
if resEachEntryFuncErr != nil {
glog.ErrorfCtx(ctx, "failed to process eachEntryFunc for entry %q: %v", fileName, resEachEntryFuncErr)
return lastFileName, fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", fileName, resEachEntryFuncErr)
}
if !resEachEntryFunc {
break
}
lastFileName = fileName
}

56
weed/filer/foundationdb/foundationdb_store_test.go
View File

@@ -372,16 +372,16 @@ func containsString(s, substr string) bool {
func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
// This test validates that DeleteFolderChildren always uses batching
// to safely handle large directories, regardless of transaction context
store := getTestStore(t)
defer store.Shutdown()
ctx := context.Background()
testDir := util.FullPath(fmt.Sprintf("/test_batch_delete_%d", time.Now().UnixNano()))
// Create a large directory (> 100 entries to trigger batching)
const NUM_ENTRIES = 250
t.Logf("Creating %d test entries...", NUM_ENTRIES)
for i := 0; i < NUM_ENTRIES; i++ {
entry := &filer.Entry{
@@ -397,11 +397,11 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
t.Fatalf("Failed to insert test entry %d: %v", i, err)
}
}
// Test 1: DeleteFolderChildren outside transaction should succeed
t.Run("OutsideTransaction", func(t *testing.T) {
testDir1 := util.FullPath(fmt.Sprintf("/test_batch_1_%d", time.Now().UnixNano()))
// Create entries
for i := 0; i < NUM_ENTRIES; i++ {
entry := &filer.Entry{
@@ -415,28 +415,28 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
}
store.InsertEntry(ctx, entry)
}
// Delete with batching
err := store.DeleteFolderChildren(ctx, testDir1)
if err != nil {
t.Errorf("DeleteFolderChildren outside transaction should succeed, got error: %v", err)
}
// Verify all entries deleted
var count int
store.ListDirectoryEntries(ctx, testDir1, "", true, 1000, func(entry *filer.Entry) bool {
store.ListDirectoryEntries(ctx, testDir1, "", true, 1000, func(entry *filer.Entry) (bool, error) {
count++
return true
return true, nil
})
if count != 0 {
t.Errorf("Expected all entries to be deleted, found %d", count)
}
})
// Test 2: DeleteFolderChildren with transaction context - uses its own batched transactions
t.Run("WithTransactionContext", func(t *testing.T) {
testDir2 := util.FullPath(fmt.Sprintf("/test_batch_2_%d", time.Now().UnixNano()))
// Create entries
for i := 0; i < NUM_ENTRIES; i++ {
entry := &filer.Entry{
@@ -450,38 +450,38 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
}
store.InsertEntry(ctx, entry)
}
// Start a transaction (DeleteFolderChildren will ignore it and use its own batching)
txCtx, err := store.BeginTransaction(ctx)
if err != nil {
t.Fatalf("BeginTransaction failed: %v", err)
}
// Delete large directory - should succeed with batching
err = store.DeleteFolderChildren(txCtx, testDir2)
if err != nil {
t.Errorf("DeleteFolderChildren should succeed with batching even when transaction context present, got: %v", err)
}
// Rollback transaction (DeleteFolderChildren used its own transactions, so this doesn't affect deletions)
store.RollbackTransaction(txCtx)
// Verify entries are still deleted (because DeleteFolderChildren managed its own transactions)
var count int
store.ListDirectoryEntries(ctx, testDir2, "", true, 1000, func(entry *filer.Entry) bool {
store.ListDirectoryEntries(ctx, testDir2, "", true, 1000, func(entry *filer.Entry) (bool, error) {
count++
return true
return true, nil
})
if count != 0 {
t.Errorf("Expected all entries to be deleted, found %d (DeleteFolderChildren uses its own transactions)", count)
}
})
// Test 3: Nested directories with batching
t.Run("NestedDirectories", func(t *testing.T) {
testDir3 := util.FullPath(fmt.Sprintf("/test_batch_3_%d", time.Now().UnixNano()))
// Create nested structure
for i := 0; i < 50; i++ {
// Files in root
@@ -495,7 +495,7 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
},
}
store.InsertEntry(ctx, entry)
// Subdirectory
subDir := &filer.Entry{
FullPath: util.NewFullPath(string(testDir3), fmt.Sprintf("dir_%02d", i)),
@@ -507,7 +507,7 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
},
}
store.InsertEntry(ctx, subDir)
// Files in subdirectory
for j := 0; j < 3; j++ {
subEntry := &filer.Entry{
@@ -522,24 +522,24 @@ func TestFoundationDBStore_DeleteFolderChildrenWithBatching(t *testing.T) {
store.InsertEntry(ctx, subEntry)
}
}
// Delete all with batching
err := store.DeleteFolderChildren(ctx, testDir3)
if err != nil {
t.Errorf("DeleteFolderChildren should handle nested directories, got: %v", err)
}
// Verify all deleted
var count int
store.ListDirectoryEntries(ctx, testDir3, "", true, 1000, func(entry *filer.Entry) bool {
store.ListDirectoryEntries(ctx, testDir3, "", true, 1000, func(entry *filer.Entry) (bool, error) {
count++
return true
return true, nil
})
if count != 0 {
t.Errorf("Expected all nested entries to be deleted, found %d", count)
}
})
// Cleanup
store.DeleteFolderChildren(ctx, testDir)
}