From b8236a10d114e5277bdb337fe0cd6a43f332e259 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 2 Apr 2026 16:57:20 -0700 Subject: [PATCH] perf(docker): pre-build Rust binaries to avoid 5-hour QEMU emulation Cross-compile Rust volume server natively for amd64/arm64 using musl targets in a separate job, then inject pre-built binaries into the Docker build. This replaces the ~5-hour QEMU-emulated cargo build with ~15 minutes of native cross-compilation. The Dockerfile falls back to building from source when no pre-built binary is found, preserving local build compatibility. --- .../workflows/container_release_unified.yml | 109 +++++++++++++++++- docker/Dockerfile.go_build | 14 ++- docker/weed-volume-prebuilt/.gitkeep | 0 3 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 docker/weed-volume-prebuilt/.gitkeep diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 391002a64..8218aaba9 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -39,7 +39,80 @@ concurrency: cancel-in-progress: false jobs: + + # ── Pre-build Rust volume server binaries natively ────────────────── + # Cross-compiles for amd64 and arm64 without QEMU, turning a 5-hour + # emulated cargo build into ~15 minutes of native compilation. + build-rust-binaries: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + arch: amd64 + - target: aarch64-unknown-linux-musl + arch: arm64 + cross: true + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install musl tools (amd64) + if: ${{ !matrix.cross }} + run: sudo apt-get install -y musl-tools + + - name: Install cross-compilation tools (arm64) + if: matrix.cross + run: | + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-docker-${{ matrix.target }}- + + - name: Build large-disk variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }} + + - name: Build normal variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: rust-volume-${{ matrix.arch }} + path: | + weed-volume-large-disk-${{ matrix.arch }} + weed-volume-normal-${{ matrix.arch }} + + # ── Build Docker containers ───────────────────────────────────────── build: + needs: [build-rust-binaries] runs-on: ubuntu-latest strategy: # Build sequentially to avoid rate limits @@ -52,20 +125,23 @@ jobs: dockerfile: ./docker/Dockerfile.go_build build_args: "" tag_suffix: "" - - # Large disk - multi-arch + rust_variant: normal + + # Large disk - multi-arch - variant: large_disk platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386 dockerfile: ./docker/Dockerfile.go_build build_args: TAGS=5BytesOffset tag_suffix: _large_disk - + rust_variant: large-disk + # Full tags - multi-arch - variant: full platforms: linux/amd64,linux/arm64 dockerfile: ./docker/Dockerfile.go_build build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb tag_suffix: _full + rust_variant: normal # Large disk + full tags - multi-arch - variant: large_disk_full @@ -73,19 +149,42 @@ jobs: dockerfile: ./docker/Dockerfile.go_build build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb tag_suffix: _large_disk_full - + rust_variant: large-disk + # RocksDB large disk - amd64 only - variant: rocksdb platforms: linux/amd64 dockerfile: ./docker/Dockerfile.rocksdb_large build_args: "" tag_suffix: _large_disk_rocksdb + rust_variant: large-disk steps: - name: Checkout if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/checkout@v6 - + + - name: Download pre-built Rust binaries + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: actions/download-artifact@v4 + with: + pattern: rust-volume-* + merge-multiple: true + path: ./rust-bins + + - name: Place Rust binaries in Docker context + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + run: | + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-${{ matrix.rust_variant }}-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ + - name: Free Disk Space if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant run: | diff --git a/docker/Dockerfile.go_build b/docker/Dockerfile.go_build index daf76af43..1f7777d72 100644 --- a/docker/Dockerfile.go_build +++ b/docker/Dockerfile.go_build @@ -16,15 +16,21 @@ RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \ && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util/version.COMMIT=$(git rev-parse --short HEAD)" \ && CGO_ENABLED=0 go install -tags "$TAGS" -ldflags "-extldflags -static ${LDFLAGS}" -# Rust volume server builder. Alpine packages avoid depending on the -# upstream rust:alpine manifest list, which no longer includes linux/386. +# Rust volume server: use pre-built binary from CI when available (placed in +# weed-volume-prebuilt/ by the build-rust-binaries job), otherwise compile +# from source. Pre-building avoids a multi-hour QEMU-emulated cargo build +# for non-native architectures. FROM alpine:3.23 as rust_builder ARG TARGETARCH +ARG TAGS +COPY weed-volume-prebuilt/ /prebuilt/ COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/seaweed-volume /build/seaweed-volume COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/weed /build/weed WORKDIR /build/seaweed-volume -ARG TAGS -RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \ +RUN if [ -f "/prebuilt/weed-volume-${TARGETARCH}" ]; then \ + echo "Using pre-built Rust binary for ${TARGETARCH}" && \ + cp "/prebuilt/weed-volume-${TARGETARCH}" /weed-volume; \ + elif [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \ apk add --no-cache musl-dev openssl-dev protobuf-dev git rust cargo; \ if [ "$TAGS" = "5BytesOffset" ]; then \ cargo build --release; \ diff --git a/docker/weed-volume-prebuilt/.gitkeep b/docker/weed-volume-prebuilt/.gitkeep new file mode 100644 index 000000000..e69de29bb