filer store: add foundationdb (#7178)

* add foundationdb * Update foundationdb_store.go * fix * apply the patch * avoid panic on error * address comments * remove extra data * address comments * adds more debug messages * fix range listing * delete with prefix range; list with right start key * fix docker files * use the more idiomatic FoundationDB KeySelectors * address comments * proper errors * fix API versions * more efficient * recursive deletion * clean up * clean up * pagination, one transaction for deletion * error checking * Use fdb.Strinc() to compute the lexicographically next string and create a proper range * fix docker * Update README.md * delete in batches * delete in batches * fix build * add foundationdb build * Updated FoundationDB Version * Fixed glibc/musl Incompatibility (Alpine → Debian) * Update container_foundationdb_version.yml * build SeaweedFS * build tag * address comments * separate transaction * address comments * fix build * empty vs no data * fixes * add go test * Install FoundationDB client libraries * nil compare
2025-11-19 20:06:57 -08:00
parent 8be9e258fc
commit c6b6ea40e6
33 changed files with 5815 additions and 0 deletions
--- a/docker/Dockerfile.foundationdb_large
+++ b/docker/Dockerfile.foundationdb_large
@@ -0,0 +1,131 @@
+FROM golang:1.24 AS builder
+
+RUN apt-get update && \
+    apt-get install -y build-essential wget ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+ARG FDB_VERSION=7.4.5
+ENV FDB_VERSION=${FDB_VERSION}
+ARG TARGETARCH
+
+# Install FoundationDB client libraries with SHA256 checksum verification
+# Known SHA256 checksums for FoundationDB client packages (verified 2025-01-19)
+# To add checksums for new versions: run docker/get_fdb_checksum.sh <version> <arch>
+RUN cd /tmp && \
+    case "${TARGETARCH}" in \
+        "amd64") FDB_ARCH="amd64"; PACKAGE_ARCH="amd64" ;; \
+        "arm64") FDB_ARCH="arm64"; PACKAGE_ARCH="aarch64" ;; \
+        *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \
+    esac && \
+    case "${FDB_VERSION}_${FDB_ARCH}" in \
+        "7.4.5_amd64") \
+            EXPECTED_SHA256="eea6b98cf386a0848655b2e196d18633662a7440a7ee061c10e32153c7e7e112" ;; \
+        "7.4.5_arm64") \
+            EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
+        "7.3.43_amd64") \
+            EXPECTED_SHA256="c3fa0a59c7355b914a1455dac909238d5ea3b6c6bc7b530af8597e6487c1651a" ;; \
+        "7.3.43_arm64") \
+            echo "ERROR: FoundationDB ${FDB_VERSION} does not publish arm64 client packages." >&2; \
+            echo "Please upgrade to 7.4.5+ when targeting arm64." >&2; \
+            exit 1 ;; \
+        *) \
+            echo "ERROR: No checksum available for FDB version ${FDB_VERSION} on ${FDB_ARCH}" >&2; \
+            echo "This is a security requirement. To add verification:" >&2; \
+            echo "  1. Run: docker/get_fdb_checksum.sh ${FDB_VERSION} ${FDB_ARCH}" >&2; \
+            echo "  2. Add the checksum to this Dockerfile" >&2; \
+            echo "Refusing to proceed without checksum verification." >&2; \
+            exit 1 ;; \
+    esac && \
+    PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb" && \
+    wget --timeout=30 --tries=3 https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE} && \
+    echo "${EXPECTED_SHA256}  ${PACKAGE}" | sha256sum -c - || \
+        (echo "ERROR: Checksum verification failed for FoundationDB ${FDB_VERSION} (${FDB_ARCH})" >&2; \
+         echo "Expected: ${EXPECTED_SHA256}" >&2; \
+         echo "This indicates either a corrupted download or potential tampering." >&2; \
+         exit 1) && \
+    dpkg -i ${PACKAGE} && \
+    rm ${PACKAGE}
+
+# Set up FoundationDB environment variables for CGO
+ENV CGO_CFLAGS="-I/usr/include/foundationdb"
+ENV CGO_LDFLAGS="-lfdb_c"
+
+# build SeaweedFS sources; prefer local context but fall back to git clone if context only has docker files
+ARG SOURCE_REF=master
+WORKDIR /go/src/github.com/seaweedfs/seaweedfs
+COPY . .
+RUN set -euo pipefail && \
+    if [ ! -d weed ]; then \
+        echo "Local build context does not include SeaweedFS sources; cloning ${SOURCE_REF}" >&2; \
+        mkdir -p /tmp/local-context && cp -a /go/src/github.com/seaweedfs/seaweedfs/. /tmp/local-context && \
+        cd / && rm -rf /go/src/github.com/seaweedfs/seaweedfs && \
+        git clone --depth 1 --branch ${SOURCE_REF} https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs && \
+        cp -a /tmp/local-context/. /go/src/github.com/seaweedfs/seaweedfs/docker/ && \
+        rm -rf /tmp/local-context && \
+        cd /go/src/github.com/seaweedfs/seaweedfs; \
+    fi && \
+    cd weed \
+  && COMMIT_SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") \
+  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=${COMMIT_SHA}" \
+  && go install -tags "5BytesOffset foundationdb" -ldflags "${LDFLAGS}"
+
+
+FROM debian:bookworm-slim AS final
+LABEL author="Chris Lu"
+
+# Install runtime dependencies first
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    ca-certificates \
+    fuse \
+    wget && \
+    rm -rf /var/lib/apt/lists/*
+
+# Reuse FoundationDB artifacts installed during the build stage
+COPY --from=builder /usr/lib/libfdb_c* /usr/lib/
+COPY --from=builder /usr/lib/foundationdb /usr/lib/foundationdb
+COPY --from=builder /usr/bin/fdb* /usr/bin/
+RUN ldconfig
+
+# Copy SeaweedFS binary and configuration
+COPY --from=builder /go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer_foundationdb.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+
+# Create non-root user
+RUN groupadd -g 1000 seaweed && \
+    useradd -u 1000 -g seaweed -s /bin/bash -m seaweed
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
+
+# Create data directory and set proper ownership for seaweed user
+RUN mkdir -p /data && \
+    chown -R seaweed:seaweed /data && \
+    chown -R seaweed:seaweed /etc/seaweedfs && \
+    chmod 755 /entrypoint.sh
+
+VOLUME /data
+
+WORKDIR /data
+
+# Switch to non-root user
+USER seaweed
+
+ENTRYPOINT ["/entrypoint.sh"]
+
--- a/docker/filer_foundationdb.toml
+++ b/docker/filer_foundationdb.toml
@@ -0,0 +1,19 @@
+[filer.options]
+# with http DELETE, by default the filer would check whether a folder is empty.
+# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
+recursive_delete = false
+
+####################################################
+# FoundationDB store
+####################################################
+[foundationdb]
+enabled = true
+cluster_file = "/etc/foundationdb/fdb.cluster"
+api_version = 740
+# Optional: timeout for FDB operations (default: 10s)
+# timeout = "10s"
+# Optional: max retry delay for retryable errors (default: 1s)
+# max_retry_delay = "1s"
+# Optional: directory prefix for storing SeaweedFS data (default: "seaweedfs")
+# directory_prefix = "seaweedfs"
+
--- a/docker/get_fdb_checksum.sh
+++ b/docker/get_fdb_checksum.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+# Helper script to get SHA256 checksum for FoundationDB client package
+# Usage: ./get_fdb_checksum.sh <version> [arch]
+# Example: ./get_fdb_checksum.sh 7.4.5 amd64
+# Example: ./get_fdb_checksum.sh 7.4.5 arm64
+
+set -euo pipefail
+
+if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+    echo "Usage: $0 <fdb_version> [arch]" >&2
+    echo "Example: $0 7.4.5" >&2
+    echo "Example: $0 7.4.5 arm64" >&2
+    exit 1
+fi
+
+FDB_VERSION="$1"
+FDB_ARCH="${2:-amd64}"
+
+case "$FDB_ARCH" in
+    "amd64")
+        CANONICAL_ARCH="amd64"
+        PACKAGE_ARCH="amd64"
+        ;;
+    "arm64"|"aarch64")
+        CANONICAL_ARCH="arm64"
+        PACKAGE_ARCH="aarch64"
+        ;;
+    *)
+        echo "Error: Architecture must be 'amd64', 'arm64', or 'aarch64'" >&2
+        exit 1
+        ;;
+esac
+
+PACKAGE="foundationdb-clients_${FDB_VERSION}-1_${PACKAGE_ARCH}.deb"
+URL="https://github.com/apple/foundationdb/releases/download/${FDB_VERSION}/${PACKAGE}"
+
+echo "Downloading FoundationDB ${FDB_VERSION} client package for ${FDB_ARCH}..."
+echo "URL: ${URL}"
+echo ""
+
+# Download to temp directory
+TEMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TEMP_DIR}"' EXIT
+
+cd "${TEMP_DIR}"
+if wget --timeout=30 --tries=3 -q "${URL}"; then
+    CHECKSUM=$(sha256sum "${PACKAGE}" | awk '{print $1}')
+    echo "✓ Download successful"
+    echo ""
+    echo "SHA256 Checksum:"
+    echo "${CHECKSUM}"
+    echo ""
+    echo "Add this to Dockerfile.foundationdb_large:"
+    echo "    \"${FDB_VERSION}_${CANONICAL_ARCH}\") \\"
+    echo "        EXPECTED_SHA256=\"${CHECKSUM}\" ;; \\"
+else
+    echo "✗ Failed to download package from ${URL}" >&2
+    echo "Please verify the version number, architecture, and URL" >&2
+    exit 1
+fi
+