From e469a56b62df44e37b1651f606cc8760524ee90a Mon Sep 17 00:00:00 2001 From: phamgialinhlx Date: Thu, 30 Apr 2026 16:36:21 +0000 Subject: [PATCH 1/3] build(python): publish manylinux_2_28 wheels for broader glibc compatibility Build released Linux wheels inside a PyPA manylinux_2_28 container so the resulting binary runs on any Linux distribution with glibc >= 2.28 (RHEL 8, Debian 10+, Ubuntu 18.04+). The previous native build on the noble (glibc 2.39) CI image produced manylinux_2_39 wheels that uv refused to install on common LTS hosts such as Ubuntu 22.04. - Add deploy/docker/Dockerfile.python-wheels-linux based on quay.io/pypa/manylinux_2_28_{x86_64,aarch64}. - Add build:python:wheel:linux:docker mise task and per-arch aliases. Re-point python:build:linux:{amd64,arm64} (used by release workflows) to the Docker-based path. The legacy native build:python:wheel:linux task remains for fast local iteration. - Mount the host docker socket and set up buildx in the build-python-wheels-linux job in release-tag.yml and release-dev.yml, mirroring the existing macOS wheel job. - Update architecture/build-containers.md to describe the new flow. Signed-off-by: phamgialinhlx --- .github/workflows/release-dev.yml | 5 + .github/workflows/release-tag.yml | 5 + deploy/docker/Dockerfile.python-wheels-linux | 119 +++++++++++++++++++ mise.lock | 1 + tasks/python.toml | 88 ++++++++++++-- 5 files changed, 209 insertions(+), 9 deletions(-) create mode 100644 deploy/docker/Dockerfile.python-wheels-linux diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 1114a2b74..4a49bd9d4 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -119,6 +119,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: dev @@ -127,6 +129,9 @@ jobs: with: fetch-depth: 0 + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx + - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index ae842494b..14230be08 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -148,6 +148,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} @@ -157,6 +159,9 @@ jobs: ref: ${{ inputs.tag || github.ref }} fetch-depth: 0 + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx + - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" diff --git a/deploy/docker/Dockerfile.python-wheels-linux b/deploy/docker/Dockerfile.python-wheels-linux new file mode 100644 index 000000000..015f19c2c --- /dev/null +++ b/deploy/docker/Dockerfile.python-wheels-linux @@ -0,0 +1,119 @@ +# syntax=docker/dockerfile:1.6 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Build the Linux Python wheel inside a PyPA manylinux_2_28 container so the +# resulting binary is compatible with any Linux distribution shipping glibc +# >= 2.28 (RHEL 8, Ubuntu 18.04+, Debian 10+). The host CI runner is on noble +# (glibc 2.39), which previously produced manylinux_2_39 wheels that uv refuses +# to install on Ubuntu 22.04 / Debian 11 (glibc 2.31 / 2.35). + +ARG TARGETARCH +ARG MANYLINUX_AMD64_IMAGE=quay.io/pypa/manylinux_2_28_x86_64:latest +ARG MANYLINUX_ARM64_IMAGE=quay.io/pypa/manylinux_2_28_aarch64:latest +ARG PYTHON_VERSION=cp312-cp312 +ARG RUST_VERSION=1.95.0 + +# Selector stages — Docker resolves only the matching one based on TARGETARCH. +FROM ${MANYLINUX_AMD64_IMAGE} AS base-amd64 +FROM ${MANYLINUX_ARM64_IMAGE} AS base-arm64 +FROM base-${TARGETARCH} AS builder + +ARG TARGETARCH +ARG PYTHON_VERSION +ARG RUST_VERSION +ARG CARGO_TARGET_CACHE_SCOPE=default + +ENV PATH="/opt/python/${PYTHON_VERSION}/bin:/root/.cargo/bin:${PATH}" + +# manylinux_2_28 ships gcc-toolset-14, cmake, and patchelf. We add clang for +# bindgen-driven crates (libclang-dev equivalent) and openssl-devel for any +# Rust crates that link against the system libssl during dependency resolution. +RUN dnf install -y --setopt=install_weak_deps=False \ + clang \ + llvm-devel \ + openssl-devel \ + perl-core \ + perl-IPC-Cmd \ + && dnf clean all + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain ${RUST_VERSION} --profile minimal +RUN python -m pip install --no-cache-dir maturin + +WORKDIR /build + +# Copy dependency manifests first for better layer caching. +COPY Cargo.toml Cargo.lock ./ +COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml +COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml +COPY crates/openshell-ocsf/Cargo.toml crates/openshell-ocsf/Cargo.toml +COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml +COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml +COPY crates/openshell-sandbox/Cargo.toml crates/openshell-sandbox/Cargo.toml +COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml +COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml +COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml +COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml +COPY crates/openshell-tui/Cargo.toml crates/openshell-tui/Cargo.toml +COPY crates/openshell-core/build.rs crates/openshell-core/build.rs +COPY proto/ proto/ + +# Create dummy source files to build dependencies. +RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src \ + crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src \ + crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src \ + crates/openshell-bootstrap/src crates/openshell-tui/src && \ + echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ + echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ + echo "fn main() {}" > crates/openshell-server/src/main.rs && \ + touch crates/openshell-core/src/lib.rs && \ + touch crates/openshell-ocsf/src/lib.rs && \ + touch crates/openshell-providers/src/lib.rs && \ + touch crates/openshell-router/src/lib.rs && \ + touch crates/openshell-bootstrap/src/lib.rs && \ + touch crates/openshell-policy/src/lib.rs && \ + touch crates/openshell-prover/src/lib.rs && \ + touch crates/openshell-tui/src/lib.rs + +# Warm the dependency build (cached unless Cargo.toml/lock changes). +RUN --mount=type=cache,id=cargo-registry-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ + --mount=type=cache,id=cargo-git-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ + --mount=type=cache,id=cargo-target-python-wheels-linux-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + cargo build --release -p openshell-cli --features bundled-z3 2>/dev/null || true + +# Copy actual source code and Python packaging files. +COPY crates/ crates/ +COPY pyproject.toml README.md ./ +COPY python/ python/ + +# Touch source files so cargo rebuilds them (not the cached dummy). +RUN touch crates/openshell-cli/src/main.rs \ + crates/openshell-cli/src/lib.rs \ + crates/openshell-bootstrap/src/lib.rs \ + crates/openshell-core/src/lib.rs \ + crates/openshell-providers/src/lib.rs \ + crates/openshell-router/src/lib.rs \ + crates/openshell-sandbox/src/main.rs \ + crates/openshell-server/src/main.rs \ + crates/openshell-core/build.rs \ + proto/*.proto + +# Declare version ARGs here (not earlier) so the git-hash-bearing values do not +# invalidate the expensive dependency-build layers above on every commit. +ARG OPENSHELL_CARGO_VERSION +ARG OPENSHELL_IMAGE_TAG +RUN --mount=type=cache,id=cargo-registry-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ + --mount=type=cache,id=cargo-git-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ + --mount=type=cache,id=cargo-target-python-wheels-linux-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ + fi && \ + maturin build --release --features bundled-z3 \ + --compatibility manylinux_2_28 \ + --out /wheels && \ + ls -la /wheels/*.whl + +FROM scratch AS wheels +COPY --from=builder /wheels/*.whl / diff --git a/mise.lock b/mise.lock index 40050e701..6f1d1fda0 100644 --- a/mise.lock +++ b/mise.lock @@ -307,6 +307,7 @@ url = "https://ziglang.org/download/0.14.1/zig-aarch64-linux-0.14.1.tar.xz" [tools.zig."platforms.linux-x64"] checksum = "sha256:24aeeec8af16c381934a6cd7d95c807a8cb2cf7df9fa40d359aa884195c4716c" url = "https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz" +provenance = "minisign" [tools.zig."platforms.macos-arm64"] checksum = "sha256:39f3dc5e79c22088ce878edc821dedb4ca5a1cd9f5ef915e9b3cc3053e8faefa" diff --git a/tasks/python.toml b/tasks/python.toml index b95d96671..1dd7f3a85 100644 --- a/tasks/python.toml +++ b/tasks/python.toml @@ -83,23 +83,93 @@ ls -la "$WHEEL_OUTPUT_DIR"/*.whl hide = true ["build:python:wheel:linux:amd64"] -description = "Build Python wheel for Linux amd64 natively" +description = "Build Python wheel for Linux amd64 natively (host glibc; non-portable)" depends = ["EXPECTED_HOST_ARCH=amd64 WHEEL_OUTPUT_DIR=target/wheels/linux-amd64 build:python:wheel:linux"] hide = true -["python:build:linux:amd64"] -description = "Alias for build:python:wheel:linux:amd64" -depends = ["build:python:wheel:linux:amd64"] -hide = true - ["build:python:wheel:linux:arm64"] -description = "Build Python wheel for Linux arm64 natively" +description = "Build Python wheel for Linux arm64 natively (host glibc; non-portable)" depends = ["EXPECTED_HOST_ARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux"] hide = true +["build:python:wheel:linux:docker"] +description = "Build a portable manylinux_2_28 Python wheel via Docker (glibc >= 2.28)" +depends = ["python:proto"] +run = """ +#!/usr/bin/env bash +set -euo pipefail + +source tasks/scripts/container-engine.sh + +WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:?Set WHEEL_OUTPUT_DIR to a per-platform wheel output directory} +TARGETARCH=${TARGETARCH:?Set TARGETARCH to amd64 or arm64} + +sha256_16() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print substr($1, 1, 16)}' + else + shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}' + fi +} + +sha256_16_stdin() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print substr($1, 1, 16)}' + else + shasum -a 256 | awk '{print substr($1, 1, 16)}' + fi +} + +CARGO_VERSION=${OPENSHELL_CARGO_VERSION:-} +if [ -z "$CARGO_VERSION" ] && [ -n "${CI:-}" ]; then + CARGO_VERSION=$(uv run python tasks/scripts/release.py get-version --cargo) +fi + +LOCK_HASH=$(sha256_16 Cargo.lock) +RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-rustup-1.95.0} +CACHE_SCOPE_INPUT="v1|python-wheels-linux-${TARGETARCH}|manylinux_2_28|${LOCK_HASH}|${RUST_SCOPE}" +CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin) + +rm -rf "$WHEEL_OUTPUT_DIR" +mkdir -p "$WHEEL_OUTPUT_DIR" + +ce build \ + -f deploy/docker/Dockerfile.python-wheels-linux \ + --target wheels \ + --build-arg "TARGETARCH=${TARGETARCH}" \ + --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \ + ${CARGO_VERSION:+--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}"} \ + ${OPENSHELL_IMAGE_TAG:+--build-arg "OPENSHELL_IMAGE_TAG=${OPENSHELL_IMAGE_TAG}"} \ + --output "type=local,dest=${WHEEL_OUTPUT_DIR}" \ + . + +ls -la "$WHEEL_OUTPUT_DIR"/*.whl +""" +hide = true + +["build:python:wheel:linux:amd64:docker"] +description = "Build portable manylinux_2_28 wheel for Linux amd64" +depends = ["TARGETARCH=amd64 WHEEL_OUTPUT_DIR=target/wheels/linux-amd64 build:python:wheel:linux:docker"] +hide = true + +["build:python:wheel:linux:arm64:docker"] +description = "Build portable manylinux_2_28 wheel for Linux arm64" +depends = ["TARGETARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux:docker"] +hide = true + +# Release-pipeline aliases. These produce manylinux_2_28-tagged wheels via +# Docker so the published artifacts install on any glibc >= 2.28 host (RHEL 8, +# Ubuntu 18.04+, Debian 10+). The native `build:python:wheel:linux:amd64` / +# `build:python:wheel:linux:arm64` tasks remain available for fast local +# iteration and produce wheels tagged for the host glibc only. +["python:build:linux:amd64"] +description = "Build portable manylinux_2_28 wheel for Linux amd64 (release path)" +depends = ["build:python:wheel:linux:amd64:docker"] +hide = true + ["python:build:linux:arm64"] -description = "Alias for build:python:wheel:linux:arm64" -depends = ["build:python:wheel:linux:arm64"] +description = "Build portable manylinux_2_28 wheel for Linux arm64 (release path)" +depends = ["build:python:wheel:linux:arm64:docker"] hide = true ["build:python:wheel:macos"] From 6722d64b82ef595b84605cc96774397bfea9a582 Mon Sep 17 00:00:00 2001 From: phamgialinhlx Date: Thu, 21 May 2026 04:06:23 +0000 Subject: [PATCH 2/3] ci(python): replace custom Dockerfile with PyO3/maturin-action for Linux wheels --- .github/workflows/release-dev.yml | 52 +++----- .github/workflows/release-tag.yml | 52 +++----- deploy/docker/Dockerfile.python-wheels-linux | 119 ------------------- tasks/python.toml | 81 ++----------- 4 files changed, 45 insertions(+), 259 deletions(-) delete mode 100644 deploy/docker/Dockerfile.python-wheels-linux diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 4a49bd9d4..6d0cb33e9 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -104,58 +104,44 @@ jobs: - arch: amd64 runner: linux-amd64-cpu8 artifact: linux-amd64 - task: python:build:linux:amd64 - output_path: target/wheels/linux-amd64/*.whl + target: x86_64-unknown-linux-gnu - arch: arm64 runner: linux-arm64-cpu8 artifact: linux-arm64 - task: python:build:linux:arm64 - output_path: target/wheels/linux-arm64/*.whl + target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} timeout-minutes: 120 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: dev steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: fetch-depth: 0 - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx + - name: Set up mise + uses: jdx/mise-action@v2 - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Generate Python protobuf stubs + run: uv sync --group dev && mise run python:proto - - name: Sync Python dependencies - run: uv sync + - name: Patch workspace version + if: needs.compute-versions.outputs.cargo_version != '' + run: | + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "${{ needs.compute-versions.outputs.cargo_version }}"/}' Cargo.toml - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 + - name: Build Python wheel + uses: PyO3/maturin-action@v1 with: - shared-key: python-wheel-linux-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Build Python wheels - run: | - set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }} - ls -la ${{ matrix.output_path }} + target: ${{ matrix.target }} + manylinux: 2_28 + args: --release --features bundled-z3 --compatibility manylinux_2_28 --out dist + before-script-linux: | + dnf install -y --setopt=install_weak_deps=False \ + clang llvm-devel openssl-devel perl-core perl-IPC-Cmd - name: Upload wheel artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: python-wheels-${{ matrix.artifact }} - path: ${{ matrix.output_path }} + path: dist/*.whl retention-days: 5 build-python-wheel-macos: diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 14230be08..c1957d970 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -133,59 +133,45 @@ jobs: - arch: amd64 runner: linux-amd64-cpu8 artifact: linux-amd64 - task: python:build:linux:amd64 - output_path: target/wheels/linux-amd64/*.whl + target: x86_64-unknown-linux-gnu - arch: arm64 runner: linux-arm64-cpu8 artifact: linux-arm64 - task: python:build:linux:arm64 - output_path: target/wheels/linux-arm64/*.whl + target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} timeout-minutes: 120 - container: - image: ghcr.io/nvidia/openshell/ci:latest - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: --privileged - volumes: - - /var/run/docker.sock:/var/run/docker.sock - env: - MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENSHELL_IMAGE_TAG: ${{ needs.compute-versions.outputs.semver }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: ref: ${{ inputs.tag || github.ref }} fetch-depth: 0 - - name: Set up Docker Buildx - uses: ./.github/actions/setup-buildx + - name: Set up mise + uses: jdx/mise-action@v2 - - name: Mark workspace safe for git - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Generate Python protobuf stubs + run: uv sync --group dev && mise run python:proto - - name: Sync Python dependencies - run: uv sync + - name: Patch workspace version + if: needs.compute-versions.outputs.cargo_version != '' + run: | + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "${{ needs.compute-versions.outputs.cargo_version }}"/}' Cargo.toml - - name: Cache Rust target and registry - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 + - name: Build Python wheel + uses: PyO3/maturin-action@v1 with: - shared-key: python-wheel-linux-${{ matrix.arch }} - cache-directories: .cache/sccache - cache-targets: "true" - - - name: Build Python wheels - run: | - set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run ${{ matrix.task }} - ls -la ${{ matrix.output_path }} + target: ${{ matrix.target }} + manylinux: 2_28 + args: --release --features bundled-z3 --compatibility manylinux_2_28 --out dist + before-script-linux: | + dnf install -y --setopt=install_weak_deps=False \ + clang llvm-devel openssl-devel perl-core perl-IPC-Cmd - name: Upload wheel artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: python-wheels-${{ matrix.artifact }} - path: ${{ matrix.output_path }} + path: dist/*.whl retention-days: 5 build-python-wheel-macos: diff --git a/deploy/docker/Dockerfile.python-wheels-linux b/deploy/docker/Dockerfile.python-wheels-linux deleted file mode 100644 index 015f19c2c..000000000 --- a/deploy/docker/Dockerfile.python-wheels-linux +++ /dev/null @@ -1,119 +0,0 @@ -# syntax=docker/dockerfile:1.6 - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Build the Linux Python wheel inside a PyPA manylinux_2_28 container so the -# resulting binary is compatible with any Linux distribution shipping glibc -# >= 2.28 (RHEL 8, Ubuntu 18.04+, Debian 10+). The host CI runner is on noble -# (glibc 2.39), which previously produced manylinux_2_39 wheels that uv refuses -# to install on Ubuntu 22.04 / Debian 11 (glibc 2.31 / 2.35). - -ARG TARGETARCH -ARG MANYLINUX_AMD64_IMAGE=quay.io/pypa/manylinux_2_28_x86_64:latest -ARG MANYLINUX_ARM64_IMAGE=quay.io/pypa/manylinux_2_28_aarch64:latest -ARG PYTHON_VERSION=cp312-cp312 -ARG RUST_VERSION=1.95.0 - -# Selector stages — Docker resolves only the matching one based on TARGETARCH. -FROM ${MANYLINUX_AMD64_IMAGE} AS base-amd64 -FROM ${MANYLINUX_ARM64_IMAGE} AS base-arm64 -FROM base-${TARGETARCH} AS builder - -ARG TARGETARCH -ARG PYTHON_VERSION -ARG RUST_VERSION -ARG CARGO_TARGET_CACHE_SCOPE=default - -ENV PATH="/opt/python/${PYTHON_VERSION}/bin:/root/.cargo/bin:${PATH}" - -# manylinux_2_28 ships gcc-toolset-14, cmake, and patchelf. We add clang for -# bindgen-driven crates (libclang-dev equivalent) and openssl-devel for any -# Rust crates that link against the system libssl during dependency resolution. -RUN dnf install -y --setopt=install_weak_deps=False \ - clang \ - llvm-devel \ - openssl-devel \ - perl-core \ - perl-IPC-Cmd \ - && dnf clean all - -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- -y --default-toolchain ${RUST_VERSION} --profile minimal -RUN python -m pip install --no-cache-dir maturin - -WORKDIR /build - -# Copy dependency manifests first for better layer caching. -COPY Cargo.toml Cargo.lock ./ -COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml -COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml -COPY crates/openshell-ocsf/Cargo.toml crates/openshell-ocsf/Cargo.toml -COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml -COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml -COPY crates/openshell-sandbox/Cargo.toml crates/openshell-sandbox/Cargo.toml -COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml -COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml -COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml -COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml -COPY crates/openshell-tui/Cargo.toml crates/openshell-tui/Cargo.toml -COPY crates/openshell-core/build.rs crates/openshell-core/build.rs -COPY proto/ proto/ - -# Create dummy source files to build dependencies. -RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src \ - crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src \ - crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src \ - crates/openshell-bootstrap/src crates/openshell-tui/src && \ - echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ - echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ - echo "fn main() {}" > crates/openshell-server/src/main.rs && \ - touch crates/openshell-core/src/lib.rs && \ - touch crates/openshell-ocsf/src/lib.rs && \ - touch crates/openshell-providers/src/lib.rs && \ - touch crates/openshell-router/src/lib.rs && \ - touch crates/openshell-bootstrap/src/lib.rs && \ - touch crates/openshell-policy/src/lib.rs && \ - touch crates/openshell-prover/src/lib.rs && \ - touch crates/openshell-tui/src/lib.rs - -# Warm the dependency build (cached unless Cargo.toml/lock changes). -RUN --mount=type=cache,id=cargo-registry-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-linux-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release -p openshell-cli --features bundled-z3 2>/dev/null || true - -# Copy actual source code and Python packaging files. -COPY crates/ crates/ -COPY pyproject.toml README.md ./ -COPY python/ python/ - -# Touch source files so cargo rebuilds them (not the cached dummy). -RUN touch crates/openshell-cli/src/main.rs \ - crates/openshell-cli/src/lib.rs \ - crates/openshell-bootstrap/src/lib.rs \ - crates/openshell-core/src/lib.rs \ - crates/openshell-providers/src/lib.rs \ - crates/openshell-router/src/lib.rs \ - crates/openshell-sandbox/src/main.rs \ - crates/openshell-server/src/main.rs \ - crates/openshell-core/build.rs \ - proto/*.proto - -# Declare version ARGs here (not earlier) so the git-hash-bearing values do not -# invalidate the expensive dependency-build layers above on every commit. -ARG OPENSHELL_CARGO_VERSION -ARG OPENSHELL_IMAGE_TAG -RUN --mount=type=cache,id=cargo-registry-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-python-wheels-linux-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-python-wheels-linux-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ - sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ - fi && \ - maturin build --release --features bundled-z3 \ - --compatibility manylinux_2_28 \ - --out /wheels && \ - ls -la /wheels/*.whl - -FROM scratch AS wheels -COPY --from=builder /wheels/*.whl / diff --git a/tasks/python.toml b/tasks/python.toml index 1dd7f3a85..7daa5e1f8 100644 --- a/tasks/python.toml +++ b/tasks/python.toml @@ -92,84 +92,17 @@ description = "Build Python wheel for Linux arm64 natively (host glibc; non-port depends = ["EXPECTED_HOST_ARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux"] hide = true -["build:python:wheel:linux:docker"] -description = "Build a portable manylinux_2_28 Python wheel via Docker (glibc >= 2.28)" -depends = ["python:proto"] -run = """ -#!/usr/bin/env bash -set -euo pipefail - -source tasks/scripts/container-engine.sh - -WHEEL_OUTPUT_DIR=${WHEEL_OUTPUT_DIR:?Set WHEEL_OUTPUT_DIR to a per-platform wheel output directory} -TARGETARCH=${TARGETARCH:?Set TARGETARCH to amd64 or arm64} - -sha256_16() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "$1" | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}' - fi -} - -sha256_16_stdin() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 | awk '{print substr($1, 1, 16)}' - fi -} - -CARGO_VERSION=${OPENSHELL_CARGO_VERSION:-} -if [ -z "$CARGO_VERSION" ] && [ -n "${CI:-}" ]; then - CARGO_VERSION=$(uv run python tasks/scripts/release.py get-version --cargo) -fi - -LOCK_HASH=$(sha256_16 Cargo.lock) -RUST_SCOPE=${RUST_TOOLCHAIN_SCOPE:-rustup-1.95.0} -CACHE_SCOPE_INPUT="v1|python-wheels-linux-${TARGETARCH}|manylinux_2_28|${LOCK_HASH}|${RUST_SCOPE}" -CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin) - -rm -rf "$WHEEL_OUTPUT_DIR" -mkdir -p "$WHEEL_OUTPUT_DIR" - -ce build \ - -f deploy/docker/Dockerfile.python-wheels-linux \ - --target wheels \ - --build-arg "TARGETARCH=${TARGETARCH}" \ - --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \ - ${CARGO_VERSION:+--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}"} \ - ${OPENSHELL_IMAGE_TAG:+--build-arg "OPENSHELL_IMAGE_TAG=${OPENSHELL_IMAGE_TAG}"} \ - --output "type=local,dest=${WHEEL_OUTPUT_DIR}" \ - . - -ls -la "$WHEEL_OUTPUT_DIR"/*.whl -""" -hide = true - -["build:python:wheel:linux:amd64:docker"] -description = "Build portable manylinux_2_28 wheel for Linux amd64" -depends = ["TARGETARCH=amd64 WHEEL_OUTPUT_DIR=target/wheels/linux-amd64 build:python:wheel:linux:docker"] -hide = true - -["build:python:wheel:linux:arm64:docker"] -description = "Build portable manylinux_2_28 wheel for Linux arm64" -depends = ["TARGETARCH=arm64 WHEEL_OUTPUT_DIR=target/wheels/linux-arm64 build:python:wheel:linux:docker"] -hide = true - -# Release-pipeline aliases. These produce manylinux_2_28-tagged wheels via -# Docker so the published artifacts install on any glibc >= 2.28 host (RHEL 8, -# Ubuntu 18.04+, Debian 10+). The native `build:python:wheel:linux:amd64` / -# `build:python:wheel:linux:arm64` tasks remain available for fast local -# iteration and produce wheels tagged for the host glibc only. +# Release-pipeline aliases. CI uses PyO3/maturin-action directly (see +# .github/workflows/release-*.yml); these aliases remain for local iteration +# and produce wheels tagged for the host glibc only. ["python:build:linux:amd64"] -description = "Build portable manylinux_2_28 wheel for Linux amd64 (release path)" -depends = ["build:python:wheel:linux:amd64:docker"] +description = "Build Python wheel for Linux amd64 (local dev; CI uses maturin-action)" +depends = ["build:python:wheel:linux:amd64"] hide = true ["python:build:linux:arm64"] -description = "Build portable manylinux_2_28 wheel for Linux arm64 (release path)" -depends = ["build:python:wheel:linux:arm64:docker"] +description = "Build Python wheel for Linux arm64 (local dev; CI uses maturin-action)" +depends = ["build:python:wheel:linux:arm64"] hide = true ["build:python:wheel:macos"] From 6c07d5379992b4765f288bbc89f5828b4bc50235 Mon Sep 17 00:00:00 2001 From: phamgialinhlx Date: Thu, 21 May 2026 04:16:26 +0000 Subject: [PATCH 3/3] docs(architecture): restore build-containers.md with maturin-action updates Upstream deleted this file in #1184. Re-add it with updated Python Wheels section documenting the PyO3/maturin-action build pipeline. --- architecture/build-containers.md | 99 ++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 architecture/build-containers.md diff --git a/architecture/build-containers.md b/architecture/build-containers.md new file mode 100644 index 000000000..1be1e6d6b --- /dev/null +++ b/architecture/build-containers.md @@ -0,0 +1,99 @@ +# Container Images + +OpenShell produces two container images, both published for `linux/amd64` and `linux/arm64`. + +## Gateway (`openshell/gateway`) + +The gateway runs the control plane API server. It is deployed as a StatefulSet inside the cluster container via a bundled Helm chart. + +- **Docker target**: `gateway` in `deploy/docker/Dockerfile.images` +- **Registry**: `ghcr.io/nvidia/openshell/gateway:latest` +- **Pulled when**: Cluster startup (the Helm chart triggers the pull) +- **Entrypoint**: `openshell-gateway --port 8080` (gRPC + HTTP, mTLS) + +## Cluster (`openshell/cluster`) + +The cluster image is a single-container Kubernetes distribution that bundles the Helm charts, Kubernetes manifests, and the `openshell-sandbox` supervisor binary needed to bootstrap the control plane. + +- **Docker target**: `cluster` in `deploy/docker/Dockerfile.images` +- **Registry**: `ghcr.io/nvidia/openshell/cluster:latest` +- **Pulled when**: `openshell gateway start` + +The supervisor binary (`openshell-sandbox`) is built before the image build, staged under `deploy/docker/.build/prebuilt-binaries//`, and copied into the cluster image at `/opt/openshell/bin/openshell-sandbox`. It is exposed to sandbox pods at runtime via a read-only `hostPath` volume mount — it is not baked into sandbox images. + +## Image Build Pipeline + +`deploy/docker/Dockerfile.images` no longer compiles Rust. CI calls `.github/workflows/shadow-rust-native-build.yml` through `workflow_call` to build `openshell-gateway` or `openshell-sandbox` natively on the target architecture. `.github/workflows/docker-build.yml` downloads the resulting artifact, stages it at `deploy/docker/.build/prebuilt-binaries//`, builds the per-arch image with the local Buildx driver, and merges multi-arch pushes with `docker buildx imagetools create`. + +Local Docker builds use `tasks/scripts/stage-prebuilt-binaries.sh` through `tasks/scripts/docker-build-image.sh` before invoking Docker, so clean checkouts do not need to create the staging directory manually. + +## Standalone Gateway Binary + +OpenShell also publishes a standalone `openshell-gateway` binary as a GitHub release asset. + +- **Source crate**: `crates/openshell-server` +- **Artifact name**: `openshell-gateway-.tar.gz` +- **Targets**: `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu`, `aarch64-apple-darwin` +- **Release workflows**: `.github/workflows/release-dev.yml`, `.github/workflows/release-tag.yml` +- **Installer**: None yet. The binary is a manual-download asset. + +Both the standalone artifact and the deployed container image use the `openshell-gateway` binary. + +## Python Wheels + +OpenShell also publishes Python wheels for `linux/amd64`, `linux/arm64`, and macOS ARM64. + +- Released Linux wheels are built per-arch using `PyO3/maturin-action` with `manylinux: 2_28`. The action pulls the PyPA `manylinux_2_28` container, installs Rust from `rust-toolchain.toml`, and runs `maturin build --features bundled-z3 --compatibility manylinux_2_28`. The resulting wheels install on any Linux with glibc >= 2.28 (RHEL 8+, Ubuntu 18.04+, Debian 10+). This follows the same pattern used by ruff and uv. +- For fast local iteration, `build:python:wheel:linux:{amd64,arm64}` build natively on the host (wheels tagged for the host glibc, not portable). +- The macOS ARM64 wheel is cross-compiled with `deploy/docker/Dockerfile.python-wheels-macos` via `build:python:wheel:macos`. +- Release workflows mirror the CLI layout: a Linux matrix job for amd64/arm64, a separate macOS job, and release jobs that download the per-platform wheel artifacts directly before publishing. + +## Sandbox Images + +Sandbox images are **not built in this repository**. They are maintained in the [openshell-community](https://github.com/nvidia/openshell-community) repository and pulled from `ghcr.io/nvidia/openshell-community/sandboxes/` at runtime. + +The default sandbox image is `ghcr.io/nvidia/openshell-community/sandboxes/base:latest`. To use a named community sandbox: + +```bash +openshell sandbox create --from +``` + +This pulls `ghcr.io/nvidia/openshell-community/sandboxes/:latest`. + +## Local Development + +`mise run cluster` is the primary development command. It bootstraps a cluster if one doesn't exist, then performs incremental deploys for subsequent runs. + +The incremental deploy (`cluster-deploy-fast.sh`) fingerprints local Git changes and only rebuilds components whose files have changed: + +| Changed files | Rebuild triggered | +|---|---| +| Cargo manifests, proto definitions, prebuilt staging script | Gateway + supervisor | +| `crates/openshell-server/*`, `crates/openshell-ocsf/*`, `deploy/docker/Dockerfile.images` | Gateway | +| `crates/openshell-sandbox/*`, `crates/openshell-policy/*` | Supervisor | +| `deploy/helm/openshell/*` | Helm upgrade | + +When no local changes are detected, the command is a no-op. + +**Gateway updates** are pushed to a local registry and the StatefulSet is restarted. **Supervisor updates** are copied directly into the running cluster container via `docker cp` — new sandbox pods pick up the updated binary immediately through the hostPath mount, with no image rebuild or cluster restart required. + +Fingerprints are stored in `.cache/cluster-deploy-fast.state`. You can also target specific components explicitly: + +```bash +mise run cluster -- gateway # rebuild gateway only +mise run cluster -- supervisor # rebuild supervisor only +mise run cluster -- chart # helm upgrade only +mise run cluster -- all # rebuild everything +``` + +To validate incremental routing and BuildKit cache reuse locally, run: + +```bash +mise run cluster:test:fast-deploy-cache +``` + +The harness runs isolated scenarios in temporary git worktrees, keeps its own state and cache under `.cache/cluster-deploy-fast-test/`, and writes a Markdown summary with: + +- auto-detection checks for gateway-only, supervisor-only, shared, Helm-only, unrelated, and explicit-target changes +- cold vs warm rebuild comparisons for gateway and supervisor code changes +- container-ID invalidation coverage to verify gateway + Helm are retriggered when the cluster container changes