gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filer store: add foundationdb (#7178)

Browse Source 
* add foundationdb

* Update foundationdb_store.go

* fix

* apply the patch

* avoid panic on error

* address comments

* remove extra data

* address comments

* adds more debug messages

* fix range listing

* delete with prefix range; list with right start key

* fix docker files

* use the more idiomatic FoundationDB KeySelectors

* address comments

* proper errors

* fix API versions

* more efficient

* recursive deletion

* clean up

* clean up

* pagination, one transaction for deletion

* error checking

* Use fdb.Strinc() to compute the lexicographically next string and create a proper range

* fix docker

* Update README.md

* delete in batches

* delete in batches

* fix build

* add foundationdb build

* Updated FoundationDB Version

* Fixed glibc/musl Incompatibility (Alpine → Debian)

* Update container_foundationdb_version.yml

* build SeaweedFS

* build tag

* address comments

* separate transaction

* address comments

* fix build

* empty vs no data

* fixes

* add go test

* Install FoundationDB client libraries

* nil compare
This commit is contained in:
Chris Lu
2025-11-19 20:06:57 -08:00
committed by GitHub
parent 8be9e258fc
commit c6b6ea40e6
33 changed files with 5815 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
.github/workflows/container_foundationdb_version.yml vendored Normal file
View File

@@ -0,0 +1,168 @@
name: "docker: build foundationdb image by version"
on:
pull_request:
branches: [ master, main ]
paths:
- 'weed/filer/foundationdb/**'
- 'test/foundationdb/**'
- 'docker/Dockerfile.foundationdb_large'
- 'docker/filer_foundationdb.toml'
- '.github/workflows/container_foundationdb_version.yml'
workflow_dispatch:
inputs:
fdb_version:
description: 'FoundationDB version to build (e.g. 7.4.5)'
required: true
default: '7.4.5'
seaweedfs_ref:
description: 'SeaweedFS git tag, branch, or commit to build'
required: true
default: 'master'
image_tag:
description: 'Optional Docker tag suffix (defaults to foundationdb_<fdb>_seaweedfs_<ref>)'
required: false
default: ''
permissions:
contents: read
jobs:
build-foundationdb-image:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install FoundationDB client libraries
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y ca-certificates wget
FDB_VERSION="${{ inputs.fdb_version || '7.4.5' }}"
case "${FDB_VERSION}_amd64" in
"7.4.5_amd64") EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;;
"7.3.43_amd64") EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;;
*)
echo "Unsupported FoundationDB version ${FDB_VERSION} for CI client install" >&2
exit 1 ;;
esac
PACKAGE="foundationdb-clients_${FDB_VERSION}-1_amd64.deb"
wget --timeout=30 --tries=3 -O "${PACKAGE}" "https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
echo "${EXPECTED_SHA256} ${PACKAGE}" | sha256sum -c -
sudo dpkg -i "${PACKAGE}"
rm "${PACKAGE}"
sudo ldconfig
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Run FoundationDB tagged tests
env:
CGO_ENABLED: 1
run: |
go test ./weed/filer/foundationdb -tags foundationdb -count=1
- name: Prepare Docker tag
id: tag
env:
FDB_VERSION_INPUT: ${{ inputs.fdb_version }}
SEAWEEDFS_REF_INPUT: ${{ inputs.seaweedfs_ref }}
CUSTOM_TAG_INPUT: ${{ inputs.image_tag }}
EVENT_NAME: ${{ github.event_name }}
HEAD_REF: ${{ github.head_ref }}
REF_NAME: ${{ github.ref_name }}
run: |
set -euo pipefail
sanitize() {
local value="$1"
value="${value,,}"
value="${value// /-}"
value="${value//[^a-z0-9_.-]/-}"
value="${value#-}"
value="${value%-}"
printf '%s' "$value"
}
version="${FDB_VERSION_INPUT}"
seaweed="${SEAWEEDFS_REF_INPUT}"
tag="${CUSTOM_TAG_INPUT}"
# Use defaults for PR builds
if [ -z "$version" ]; then
version="7.4.5"
fi
if [ -z "$seaweed" ]; then
if [ "$EVENT_NAME" = "pull_request" ]; then
seaweed="${HEAD_REF}"
else
seaweed="${REF_NAME}"
fi
fi
sanitized_version="$(sanitize "$version")"
if [ -z "$sanitized_version" ]; then
echo "Unable to sanitize FoundationDB version '$version'." >&2
exit 1
fi
sanitized_seaweed="$(sanitize "$seaweed")"
if [ -z "$sanitized_seaweed" ]; then
echo "Unable to sanitize SeaweedFS ref '$seaweed'." >&2
exit 1
fi
if [ -z "$tag" ]; then
tag="foundationdb_${sanitized_version}_seaweedfs_${sanitized_seaweed}"
else
tag="$(sanitize "$tag")"
fi
if [ -z "$tag" ]; then
echo "Resulting Docker tag is empty." >&2
exit 1
fi
echo "docker_tag=$tag" >> "$GITHUB_OUTPUT"
echo "full_image=chrislusf/seaweedfs:$tag" >> "$GITHUB_OUTPUT"
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Determine branch to build
id: branch
run: |
if [ -n "${{ inputs.seaweedfs_ref }}" ]; then
echo "branch=${{ inputs.seaweedfs_ref }}" >> "$GITHUB_OUTPUT"
elif [ "${{ github.event_name }}" = "pull_request" ]; then
echo "branch=${{ github.head_ref }}" >> "$GITHUB_OUTPUT"
else
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.foundationdb_large
build-args: |
FDB_VERSION=${{ inputs.fdb_version || '7.4.5' }}
BRANCH=${{ steps.branch.outputs.branch }}
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
# Currently only building for amd64. To enable ARM64, verify package availability and add checksums.
platforms: linux/amd64
tags: ${{ steps.tag.outputs.full_image || 'seaweedfs:foundationdb-test' }}
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu

71
.github/workflows/container_release_foundationdb.yml vendored Normal file
View File

@@ -0,0 +1,71 @@
name: "docker: build release containers for foundationdb"
on:
push:
tags:
- '*'
workflow_dispatch: {}
permissions:
contents: read
jobs:
build-large-release-container_foundationdb:
runs-on: [ubuntu-latest]
steps:
-
name: Checkout
uses: actions/checkout@v4
-
name: Docker meta
id: docker_meta
uses: docker/metadata-action@v5
with:
images: |
chrislusf/seaweedfs
tags: |
type=ref,event=tag,suffix=_large_disk_foundationdb
flavor: |
latest=false
labels: |
org.opencontainers.image.title=seaweedfs
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@v3
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Determine branch to build
id: branch
run: |
if [ "${{ github.event_name }}" = "push" ] && [ -n "${{ github.ref_name }}" ]; then
echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
else
echo "branch=master" >> "$GITHUB_OUTPUT"
fi
-
name: Build
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
file: ./docker/Dockerfile.foundationdb_large
build-args: |
BRANCH=${{ steps.branch.outputs.branch }}
# Note: ARM64 support requires FoundationDB ARM64 packages which are not available for all versions
platforms: linux/amd64
tags: ${{ steps.docker_meta.outputs.tags }}
labels: ${{ steps.docker_meta.outputs.labels }}