update workflows (#1863)

diptanu · web-flow · commit f820dd5395ee · 2025-11-15T13:56:01.000-08:00
* update workflows

* fix aarch build for indexify server release packages

* fix validation error

* fix build

* fix path

* fix
diff --git a/.github/workflows/publish_indexify_server.yaml b/.github/workflows/publish_indexify_server.yaml
@@ -66,104 +66,69 @@ jobs:
           path: /tmp/release/
       - name: Name Artifacts
         run: |
-          mv /tmp/release/indexify-server-darwin-arm64/indexify-server /tmp/release/indexify-server-${{ needs.extract-version.outputs.version }}-darwin-arm64
           mv /tmp/release/indexify-server-linux-amd64/indexify-server /tmp/release/indexify-server-${{ needs.extract-version.outputs.version }}-linux-amd64
-          mv /tmp/release/indexify-server-deb-linux-amd64/indexify-server_${{ needs.extract-version.outputs.version }}-1_amd64.deb /tmp/release/indexify-server-${{ needs.extract-version.outputs.version }}-linux-amd64.deb
+          mv /tmp/release/indexify-server-linux-aarch64/indexify-server /tmp/release/indexify-server-${{ needs.extract-version.outputs.version }}-linux-aarch64
 
       - name: Create GitHub Release
-        id: create_release
         uses: softprops/action-gh-release@v2
         with:
           tag_name: 'v${{ needs.extract-version.outputs.version }}'
           prerelease: ${{ github.event.inputs.prerelease }}
           target_commitish: ${{ inputs.ref }}
-
-      - name: Upload Linux Binary
-        uses: actions/upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path:
-            /tmp/release/indexify-server-${{
-            needs.extract-version.outputs.version }}-linux-amd64
-          asset_name:
-            indexify-server-${{ needs.extract-version.outputs.version
-            }}-linux-amd64
-          asset_content_type: application/octet-stream
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Upload Linux Deb Package for amd64
-        uses: actions/upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path:
-            /tmp/release/indexify-server-${{
-            needs.extract-version.outputs.version }}-linux-amd64.deb
-          asset_name:
-            indexify-server-${{ needs.extract-version.outputs.version
-            }}-linux-amd64.deb
-          asset_content_type: application/octet-stream
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Upload macOS Binary
-        uses: actions/upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path:
-            /tmp/release/indexify-server-${{
-            needs.extract-version.outputs.version }}-darwin-arm64
-          asset_name:
-            indexify-server-${{ needs.extract-version.outputs.version
-            }}-darwin-arm64
-          asset_content_type: application/octet-stream
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: install reprepro
-        run: sudo apt-get update && sudo apt-get install -y reprepro
-
-      - name: Run reprepro
-        run: |
-          ls /tmp/release/
-          (cd .repo && reprepro includedeb buster /tmp/release/*.deb)
-          (cd .repo && reprepro includedeb focal /tmp/release/*.deb)
-          (cd .repo && reprepro includedeb jammy /tmp/release/*.deb)
-          # (cd .repo && reprepro includedeb noble /tmp/release/*.deb)
-          ls .repo/
-
-      - uses: ryand56/r2-upload-action@latest
-        with:
-          r2-account-id: ${{ secrets.R2_ACCOUNT_ID }}
-          r2-access-key-id: ${{ secrets.R2_ACCESS_KEY_ID }}
-          r2-secret-access-key: ${{ secrets.R2_SECRET_ACCESS_KEY }}
-          r2-bucket: ${{ secrets.R2_BUCKET }}
-          source-dir: server/.repo/
-          destination-dir: ./repo
+          files: |
+            /tmp/release/indexify-server-${{ needs.extract-version.outputs.version }}-linux-amd64
+            /tmp/release/indexify-server-${{ needs.extract-version.outputs.version }}-linux-aarch64
 
   build-and-push-docker-images:
     name: Build and Push Docker Images
     runs-on: namespace-profile-indexify-builder
     needs:
+      - build-release-packages
       - extract-version
       - create-release
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
 
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/docker-binaries/
+
+      - name: Prepare binaries for Docker build
+        run: |
+          # Docker buildx uses amd64 and arm64 for TARGETARCH
+          # Copy and rename binaries to match Docker's architecture naming
+          # Note: We're already in ./server directory due to defaults.run.working-directory
+          ls -la /tmp/docker-binaries/
+          cp /tmp/docker-binaries/indexify-server-linux-amd64/indexify-server ./indexify-server-amd64
+          cp /tmp/docker-binaries/indexify-server-linux-aarch64/indexify-server ./indexify-server-arm64
+          ls -la ./indexify-server-*
+          chmod +x ./indexify-server-amd64 ./indexify-server-arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - run: |
-          docker buildx build --platform=linux/amd64 --push . -t tensorlake/indexify-server:latest -f dockerfiles/Dockerfile.release_server;
 
-          tag="";
+      - name: Build and push multi-platform Docker images
+        run: |
+          # Build and push latest tag with multi-platform support
+          docker buildx build --platform=linux/amd64,linux/arm64 --push . -t tensorlake/indexify-server:latest -f dockerfiles/Dockerfile.release_server
+
+          # Build and push version tags
+          tag=""
           for i in $(echo ${{ needs.extract-version.outputs.version }} | tr '.' '\n')
           do
             if [[ $tag == "" ]]; then
-              tag="$i";
+              tag="$i"
             else
-              tag="$tag.$i";
+              tag="$tag.$i"
             fi
-            docker buildx build --platform=linux/amd64 --push . -t tensorlake/indexify-server:$tag -f dockerfiles/Dockerfile.release_server;
+            docker buildx build --platform=linux/amd64,linux/arm64 --push . -t tensorlake/indexify-server:$tag -f dockerfiles/Dockerfile.release_server
           done
diff --git a/.github/workflows/wf_build_indexify_server_release_packages.yaml b/.github/workflows/wf_build_indexify_server_release_packages.yaml
@@ -18,7 +18,7 @@ defaults:
 
 jobs:
   build-linux-amd64-package:
-    name: Build x86_64 Linux Package
+    name: Build x86_64 Linux Binary
     runs-on: ubuntu-latest-xlarge
     steps:
       - uses: actions/checkout@v4
@@ -34,34 +34,43 @@ jobs:
             ~/.cargo/registry
             ~/.cargo/git
 
-      - run: cargo install cargo-deb
       - run: cargo build --release
-      - run: cargo deb --no-build --no-strip --target x86_64-unknown-linux-gnu
 
       - uses: actions/upload-artifact@v4
         with:
           name: indexify-server-linux-amd64
           path: target/release/indexify-server
           if-no-files-found: error
 
-  build-macos-package:
-    name: Build macOS Package
-    runs-on: macos-14-xlarge
+  build-linux-arm64-package:
+    name: Build Aarch64 Linux Binary
+    runs-on: ubuntu-latest-xlarge
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ inputs.ref }}
-      - run: NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-      - run: brew install protobuf
-      - run: rustup update
-      - run: rustup toolchain install nightly --component rustfmt
-      - run: rustup target add aarch64-apple-darwin
-      - run: cargo +nightly fmt --check
-      - run: make package-ui
-      - run: cargo build --release
+
+      - run: sudo apt update && sudo apt install -y protobuf-compiler gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache-directories: |
+            target
+            ~/.cargo/registry
+            ~/.cargo/git
+
+      - run: rustup target add aarch64-unknown-linux-gnu
+
+      - name: Build for ARM64
+        env:
+          CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+          CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
+          CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+        run: cargo build --release --target aarch64-unknown-linux-gnu
+
       - uses: actions/upload-artifact@v4
         with:
-          name: indexify-server-darwin-arm64
-          path: target/release/indexify-server
+          name: indexify-server-linux-aarch64
+          path: target/aarch64-unknown-linux-gnu/release/indexify-server
           if-no-files-found: error
 
diff --git a/server/dockerfiles/Dockerfile.release_server b/server/dockerfiles/Dockerfile.release_server
@@ -1,16 +1,16 @@
 FROM ubuntu:22.04
 
-RUN apt update
-
-RUN apt install -y lsb-release ca-certificates apt-transport-https
-
-RUN update-ca-certificates
-
-RUN echo "deb [trusted=yes] https://cf-repo.diptanu-6d5.workers.dev/repo $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/indexify-repo.list && \
-    apt-get update -y && \
-    apt-get install -y indexify-server && \
-    apt-get -y clean
+RUN apt-get update && \
+    apt-get install -y ca-certificates && \
+    update-ca-certificates && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /indexify
 
-ENTRYPOINT [ "indexify-server" ]
+# The binary will be copied in during the Docker build via buildx
+# The TARGETARCH build arg is automatically set by Docker buildx
+ARG TARGETARCH
+COPY --chmod=755 indexify-server-${TARGETARCH} /indexify/indexify-server
+
+ENTRYPOINT [ "/indexify/indexify-server" ]
