From 47baf6c8410f44056af45eaf330306aadf9c3a54 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 4 Apr 2026 13:53:13 -0700 Subject: [PATCH] fix(docker): add Rust volume server pre-build to latest and dev container workflows Both container_latest.yml and container_dev.yml use Dockerfile.go_build which expects weed-volume-prebuilt/ with pre-compiled Rust binaries, but neither workflow produced them, causing COPY failures during docker build. Add build-rust-binaries jobs that natively cross-compile for amd64 and arm64, then download and place the artifacts in the Docker build context. Also fix the trivy-scan local build path in container_latest.yml. --- .github/workflows/container_dev.yml | 123 ++++++++++++++++++++----- .github/workflows/container_latest.yml | 118 +++++++++++++++++++++++- 2 files changed, 216 insertions(+), 25 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index c85a444d5..999c71727 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -9,17 +9,94 @@ permissions: jobs: + # ── Pre-build Rust volume server binaries natively ────────────────── + build-rust-binaries: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + arch: amd64 + - target: aarch64-unknown-linux-musl + arch: arm64 + cross: true + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install musl tools (amd64) + if: ${{ !matrix.cross }} + run: sudo apt-get install -y musl-tools + + - name: Install cross-compilation tools (arm64) + if: matrix.cross + run: | + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-docker-dev-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-docker-dev-${{ matrix.target }}- + + - name: Build normal variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: rust-volume-${{ matrix.arch }} + path: weed-volume-normal-${{ matrix.arch }} + build-dev-containers: + needs: [build-rust-binaries] runs-on: [ubuntu-latest] steps: - - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v2 - - - name: Docker meta + - name: Checkout + uses: actions/checkout@v6 + + - name: Download pre-built Rust binaries + uses: actions/download-artifact@v4 + with: + pattern: rust-volume-* + merge-multiple: true + path: ./rust-bins + + - name: Place Rust binaries in Docker context + run: | + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-normal-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ + + - name: Docker meta id: docker_meta - uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v3 + uses: docker/metadata-action@v6 with: images: | chrislusf/seaweedfs @@ -30,40 +107,40 @@ jobs: org.opencontainers.image.title=seaweedfs org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast! org.opencontainers.image.vendor=Chris Lu - - - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1 - - - name: Create BuildKit config + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Create BuildKit config run: | cat > /tmp/buildkitd.toml <> "$GITHUB_OUTPUT" fi + # ── Pre-build Rust volume server binaries natively ────────────────── + build-rust-binaries: + runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: x86_64-unknown-linux-musl + arch: amd64 + - target: aarch64-unknown-linux-musl + arch: arm64 + cross: true + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Install musl tools (amd64) + if: ${{ !matrix.cross }} + run: sudo apt-get install -y musl-tools + + - name: Install cross-compilation tools (arm64) + if: matrix.cross + run: | + sudo apt-get install -y gcc-aarch64-linux-gnu + echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" + + - name: Cache cargo registry and target + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-docker-${{ matrix.target }}-${{ hashFiles('seaweed-volume/Cargo.lock') }} + restore-keys: | + rust-docker-${{ matrix.target }}- + + - name: Build large-disk variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-large-disk-${{ matrix.arch }} + + - name: Build normal variant + env: + SEAWEEDFS_COMMIT: ${{ github.sha }} + run: | + cd seaweed-volume + cargo build --release --target ${{ matrix.target }} --no-default-features + cp target/${{ matrix.target }}/release/weed-volume ../weed-volume-normal-${{ matrix.arch }} + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: rust-volume-${{ matrix.arch }} + path: | + weed-volume-large-disk-${{ matrix.arch }} + weed-volume-normal-${{ matrix.arch }} + build: - needs: [setup] + needs: [setup, build-rust-binaries] runs-on: ubuntu-latest strategy: matrix: @@ -94,11 +164,32 @@ jobs: if [ "${{ matrix.variant }}" == "large_disk" ]; then echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT + echo "rust_variant=large-disk" >> $GITHUB_OUTPUT else echo "tag_suffix=" >> $GITHUB_OUTPUT echo "build_args=" >> $GITHUB_OUTPUT + echo "rust_variant=normal" >> $GITHUB_OUTPUT fi + - name: Download pre-built Rust binaries + uses: actions/download-artifact@v4 + with: + pattern: rust-volume-* + merge-multiple: true + path: ./rust-bins + + - name: Place Rust binaries in Docker context + run: | + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-${{ steps.config.outputs.rust_variant }}-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ + - name: Docker meta id: docker_meta uses: docker/metadata-action@v6 @@ -166,7 +257,7 @@ jobs: trivy-scan: runs-on: ubuntu-latest - needs: [setup, build] + needs: [setup, build, build-rust-binaries] strategy: matrix: variant: ${{ fromJSON(needs.setup.outputs.variants) }} @@ -191,6 +282,29 @@ jobs: uses: actions/checkout@v6 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + - name: Download pre-built Rust binaries for local scan + if: needs.setup.outputs.publish != 'true' + uses: actions/download-artifact@v4 + with: + pattern: rust-volume-* + merge-multiple: true + path: ./rust-bins + - name: Place Rust binaries in Docker context for local scan + if: needs.setup.outputs.publish != 'true' + run: | + rust_variant="normal" + if [ "${{ matrix.variant }}" == "large_disk" ]; then + rust_variant="large-disk" + fi + mkdir -p docker/weed-volume-prebuilt + for arch in amd64 arm64; do + src="./rust-bins/weed-volume-${rust_variant}-${arch}" + if [ -f "$src" ]; then + cp "$src" "docker/weed-volume-prebuilt/weed-volume-${arch}" + echo "Placed pre-built Rust binary for ${arch}" + fi + done + ls -la docker/weed-volume-prebuilt/ - name: Create BuildKit config for local scan build if: needs.setup.outputs.publish != 'true' run: |