gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
e5f72077ee6a5c5299f28c73a05f9477b7b54540
seaweedFS/test/foundationdb/docker-compose.arm64.yml
Chris Lu 5a03b5538f filer: improve FoundationDB performance by disabling batch by default (#7770) 
* filer: improve FoundationDB performance by disabling batch by default

This PR addresses a performance issue where FoundationDB filer was achieving
only ~757 ops/sec with 12 concurrent S3 clients, despite FDB being capable
of 17,000+ ops/sec.

Root cause: The write batcher was waiting up to 5ms for each operation to
batch, even though S3 semantics require waiting for durability confirmation.
This added artificial latency that defeated the purpose of batching.

Changes:
- Disable write batching by default (batch_enabled = false)
- Each write now commits immediately in its own transaction
- Reduce batch interval from 5ms to 1ms when batching is enabled
- Add batch_enabled config option to toggle behavior
- Improve batcher to collect available ops without blocking
- Add benchmarks comparing batch vs no-batch performance

Benchmark results (16 concurrent goroutines):
- With batch:    2,924 ops/sec (342,032 ns/op)
- Without batch: 4,625 ops/sec (216,219 ns/op)
- Improvement:   +58% faster

Configuration:
- Default: batch_enabled = false (optimal for S3 PUT latency)
- For bulk ingestion: set batch_enabled = true

Also fixes ARM64 Docker test setup (shell compatibility, fdbserver path).

* fix: address review comments - use atomic counter and remove duplicate batcher

- Use sync/atomic.Uint64 for unique filenames in concurrent benchmarks
- Remove duplicate batcher creation in createBenchmarkStoreWithBatching
  (initialize() already creates batcher when batchEnabled=true)

* fix: add realistic default values to benchmark store helper

Set directoryPrefix, timeout, and maxRetryDelay to reasonable defaults
for more realistic benchmark conditions.
2025-12-15 13:03:34 -08:00

178 lines
5.3 KiB
YAML
Raw Blame History

version: '3.9'
services:
# FoundationDB cluster nodes - ARM64 compatible
fdb1:
build:
context: .
dockerfile: Dockerfile.fdb-arm64
platforms:
- linux/arm64
platform: linux/arm64
environment:
- FDB_NETWORKING_MODE=host
- FDB_COORDINATOR_PORT=4500
- FDB_PORT=4501
ports:
- "4500:4500"
- "4501:4501"
volumes:
- fdb1_data:/var/fdb/data
- fdb_config:/var/fdb/config
networks:
- fdb_network
command: |
bash -c "
# Initialize cluster configuration
if [ ! -f /var/fdb/config/fdb.cluster ]; then
echo 'testing:testing@fdb1:4500,fdb2:4502,fdb3:4504' > /var/fdb/config/fdb.cluster
fi
# Start FDB processes
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
wait
"
fdb2:
build:
context: .
dockerfile: Dockerfile.fdb-arm64
platforms:
- linux/arm64
platform: linux/arm64
environment:
- FDB_NETWORKING_MODE=host
- FDB_COORDINATOR_PORT=4502
- FDB_PORT=4503
ports:
- "4502:4502"
- "4503:4503"
volumes:
- fdb2_data:/var/fdb/data
- fdb_config:/var/fdb/config
networks:
- fdb_network
depends_on:
- fdb1
command: |
bash -c "
# Wait for cluster file from fdb1
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
# Start FDB processes
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
wait
"
fdb3:
build:
context: .
dockerfile: Dockerfile.fdb-arm64
platforms:
- linux/arm64
platform: linux/arm64
environment:
- FDB_NETWORKING_MODE=host
- FDB_COORDINATOR_PORT=4504
- FDB_PORT=4505
ports:
- "4504:4504"
- "4505:4505"
volumes:
- fdb3_data:/var/fdb/data
- fdb_config:/var/fdb/config
networks:
- fdb_network
depends_on:
- fdb1
command: |
bash -c "
# Wait for cluster file from fdb1
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
# Start FDB processes
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
wait
"
# Initialize and configure the database
fdb-init:
build:
context: .
dockerfile: Dockerfile.fdb-arm64
platforms:
- linux/arm64
platform: linux/arm64
volumes:
- fdb_config:/var/fdb/config
networks:
- fdb_network
depends_on:
- fdb1
- fdb2
- fdb3
command: |
bash -c "
set -euo pipefail
# Wait for cluster file
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
# Wait for cluster to be ready
sleep 10
# Configure database
echo 'Initializing FoundationDB database...'
fdbcli -C /var/fdb/config/fdb.cluster --exec 'configure new single ssd'
# Wait for configuration to complete
sleep 5
# Verify cluster status
fdbcli -C /var/fdb/config/fdb.cluster --exec 'status'
echo 'FoundationDB cluster initialization complete!'
"
# SeaweedFS service with FoundationDB filer
seaweedfs:
build:
context: ../..
dockerfile: test/foundationdb/Dockerfile.build.arm64
platforms:
- linux/arm64
platform: linux/arm64
ports:
- "9333:9333"
- "19333:19333"
- "8888:8888"
- "8333:8333"
- "18888:18888"
command: "server -ip=seaweedfs -filer -master.volumeSizeLimitMB=16 -volume.max=0 -volume -volume.preStopSeconds=1 -s3 -s3.config=/etc/seaweedfs/s3.json -s3.port=8333 -s3.allowDeleteBucketNotEmpty=false"
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
- ./filer.toml:/etc/seaweedfs/filer.toml
- fdb_config:/var/fdb/config
environment:
WEED_LEVELDB2_ENABLED: "false"
WEED_FOUNDATIONDB_ENABLED: "true"
WEED_FOUNDATIONDB_CLUSTER_FILE: "/var/fdb/config/fdb.cluster"
WEED_FOUNDATIONDB_API_VERSION: "740"
WEED_FOUNDATIONDB_TIMEOUT: "5s"
WEED_FOUNDATIONDB_MAX_RETRY_DELAY: "1s"
WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
networks:
- fdb_network
depends_on:
- fdb-init
volumes:
fdb1_data:
fdb2_data:
fdb3_data:
fdb_config:
networks:
fdb_network:
driver: bridge
Reference in New Issue View Git Blame Copy Permalink