From 6b6b22f648afb796cdb83b5d3cffabacc993057f Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sat, 28 Jun 2025 10:42:44 -0500 Subject: [PATCH] Generate the matrix for building the `pythonbuild` crate --- .github/workflows/linux.yml | 49 ++++++++-------- .github/workflows/macos.yml | 44 ++++++++------- .github/workflows/windows.yml | 44 ++++++++------- ci-matrix.py | 102 +++++++++++++++++++++++++++++----- 4 files changed, 162 insertions(+), 77 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 6f73a952..8ae87e76 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -13,11 +13,14 @@ env: FORCE_COLOR: 1 jobs: - pythonbuild: - if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }} + crate-build: needs: - generate-matrix - runs-on: depot-ubuntu-22.04 + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }} + fail-fast: false + name: crate / ${{ matrix.arch }} steps: - name: Install System Dependencies run: | @@ -45,7 +48,7 @@ jobs: - name: Upload pythonbuild Executable uses: actions/upload-artifact@v4 with: - name: pythonbuild + name: ${{ matrix.crate_artifact_name }} path: target/release/pythonbuild image: @@ -55,7 +58,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(needs.generate-matrix.outputs.docker-build-matrix) }} - name: ${{ matrix.name }} + name: image / ${{ matrix.name }} runs-on: ${{ matrix.runner }} permissions: packages: write @@ -122,8 +125,8 @@ jobs: python-build-matrix-0: ${{ steps.set-matrix.outputs.python-build-matrix-0 }} python-build-matrix-1: ${{ steps.set-matrix.outputs.python-build-matrix-1 }} docker-build-matrix: ${{ steps.set-matrix.outputs.docker-build-matrix }} + crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }} any_builds: ${{ steps.set-matrix.outputs.any_builds }} - pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }} steps: - uses: actions/checkout@v4 with: @@ -139,6 +142,18 @@ jobs: LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")') echo "labels=$LABELS" >> $GITHUB_OUTPUT + - name: Check if the `pythonbuild` crate changed + id: check-pythonbuild + env: + BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} + run: | + merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Generate build matrix id: set-matrix run: | @@ -146,11 +161,13 @@ jobs: --platform linux \ --labels '${{ steps.get-labels.outputs.labels }}' \ --max-shards 2 \ + ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} \ > matrix.json echo "python-build-matrix-0=$(jq -c '."python-build"["0"]' matrix.json)" >> $GITHUB_OUTPUT echo "python-build-matrix-1=$(jq -c '."python-build"["1"]' matrix.json)" >> $GITHUB_OUTPUT echo "docker-build-matrix=$(jq -c '."docker-build"' matrix.json)" >> $GITHUB_OUTPUT + echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT # Display the matrix for debugging too cat matrix.json | jq @@ -163,22 +180,10 @@ jobs: echo "any_builds=false" >> $GITHUB_OUTPUT fi - - name: Check if the `pythonbuild` crate changed - id: check-pythonbuild - env: - BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} - run: | - merge_base=$(git merge-base HEAD "origin/${BASE_REF}") - if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - build-0: needs: - generate-matrix - - pythonbuild + - crate-build - image # Permissions used for actions/attest-build-provenance permissions: @@ -202,7 +207,7 @@ jobs: - name: Download pythonbuild uses: actions/download-artifact@v4 with: - name: pythonbuild + name: ${{ matrix.crate_artifact_name }} path: build - name: Download images @@ -278,7 +283,7 @@ jobs: build-1: needs: - generate-matrix - - pythonbuild + - crate-build - image # Permissions used for actions/attest-build-provenance permissions: @@ -302,7 +307,7 @@ jobs: - name: Download pythonbuild uses: actions/download-artifact@v4 with: - name: pythonbuild + name: ${{ matrix.crate_artifact_name }} path: build - name: Download images diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 918323d4..1199b398 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -13,11 +13,14 @@ env: FORCE_COLOR: 1 jobs: - pythonbuild: - if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }} + crate-build: needs: - generate-matrix - runs-on: depot-macos-latest + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }} + fail-fast: false + name: crate / ${{ matrix.arch }} steps: - uses: actions/checkout@v4 @@ -40,15 +43,15 @@ jobs: - name: Upload pythonbuild Executable uses: actions/upload-artifact@v4 with: - name: pythonbuild + name: ${{ matrix.crate_artifact_name }} path: target/release/pythonbuild generate-matrix: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} + crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }} any_builds: ${{ steps.set-matrix.outputs.any_builds }} - pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }} steps: - uses: actions/checkout@v4 with: @@ -64,13 +67,26 @@ jobs: LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")') echo "labels=$LABELS" >> $GITHUB_OUTPUT + - name: Check if the `pythonbuild` crate changed + id: check-pythonbuild + env: + BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} + run: | + merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Generate build matrix id: set-matrix run: | - uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json + uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json # Extract python-build matrix echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT + echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT # Display the matrix for debugging too cat matrix.json | jq @@ -83,22 +99,10 @@ jobs: echo "any_builds=false" >> $GITHUB_OUTPUT fi - - name: Check if the `pythonbuild` crate changed - id: check-pythonbuild - env: - BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} - run: | - merge_base=$(git merge-base HEAD "origin/${BASE_REF}") - if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - build: needs: - generate-matrix - - pythonbuild + - crate-build # Permissions used for actions/attest-build-provenance permissions: id-token: write @@ -121,7 +125,7 @@ jobs: - name: Download pythonbuild uses: actions/download-artifact@v4 with: - name: pythonbuild + name: ${{ matrix.crate_artifact_name }} path: build - name: Build diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 0e03dbaf..e285fac3 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -13,11 +13,14 @@ env: FORCE_COLOR: 1 jobs: - pythonbuild: - if: ${{ needs.generate-matrix.outputs.pythonbuild_changed == 'true' || needs.generate-matrix.outputs.any_builds == 'true' || github.ref == 'refs/heads/main' }} + crate-build: needs: - generate-matrix - runs-on: 'windows-2022' + runs-on: ${{ matrix.runner }} + strategy: + matrix: ${{ fromJson(needs.generate-matrix.outputs.crate-build-matrix) }} + fail-fast: false + name: crate / ${{ matrix.arch }} steps: - uses: actions/checkout@v4 @@ -40,15 +43,15 @@ jobs: - name: Upload executable uses: actions/upload-artifact@v4 with: - name: pythonbuild + name: ${{ matrix.crate_artifact_name }} path: target/release/pythonbuild.exe generate-matrix: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} + crate-build-matrix: ${{ steps.set-matrix.outputs.crate-build-matrix }} any_builds: ${{ steps.set-matrix.outputs.any_builds }} - pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }} steps: - uses: actions/checkout@v4 with: @@ -64,13 +67,26 @@ jobs: LABELS=$(echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -r 'join(",")') echo "labels=$LABELS" >> $GITHUB_OUTPUT + - name: Check if the `pythonbuild` crate changed + id: check-pythonbuild + env: + BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} + run: | + merge_base=$(git merge-base HEAD "origin/${BASE_REF}") + if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + - name: Generate build matrix id: set-matrix run: | - uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json + uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' ${{ (steps.check-pythonbuild.outputs.changed == 'true' || github.ref == 'refs/heads/main') && '--force-crate-build' || '' }} > matrix.json # Extract python-build matrix echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT + echo "crate-build-matrix=$(jq -c '."crate-build"' matrix.json)" >> $GITHUB_OUTPUT # Display the matrix for debugging too cat matrix.json | jq @@ -83,23 +99,11 @@ jobs: echo "any_builds=false" >> $GITHUB_OUTPUT fi - - name: Check if the `pythonbuild` crate changed - id: check-pythonbuild - env: - BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} - run: | - merge_base=$(git merge-base HEAD "origin/${BASE_REF}") - if git diff --quiet "${merge_base}...HEAD" -- ':src/*.rs'; then - echo "changed=false" >> "$GITHUB_OUTPUT" - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - build: timeout-minutes: 60 needs: - generate-matrix - - pythonbuild + - crate-build # Permissions used for actions/attest-build-provenance permissions: id-token: write @@ -127,7 +131,7 @@ jobs: - name: Download pythonbuild Executable uses: actions/download-artifact@v4 with: - name: pythonbuild + name: ${{ matrix.crate_artifact_name }} # We need to do this before we activate the VC++ environment or else binary packages # don't get compiled properly. diff --git a/ci-matrix.py b/ci-matrix.py index 8543a582..96059a96 100644 --- a/ci-matrix.py +++ b/ci-matrix.py @@ -19,6 +19,7 @@ CI_EXTRA_SKIP_LABELS = ["documentation"] CI_MATRIX_SIZE_LIMIT = 256 # The maximum size of a matrix in GitHub Actions + # Docker images for building toolchains and dependencies DOCKER_BUILD_IMAGES = [ {"name": "build", "arch": "x86_64"}, @@ -28,6 +29,10 @@ ] +def crate_artifact_name(platform: str, arch: str) -> str: + return f"crate-{platform}-{arch}" + + def meets_conditional_version(version: str, min_version: str) -> bool: return Version(version) >= Version(min_version) @@ -108,7 +113,7 @@ def generate_docker_matrix_entries( matrix_entries = [] for image in DOCKER_BUILD_IMAGES: # Find appropriate runner for Linux platform with the specified architecture - runner = find_runner(runners, "linux", image["arch"]) + runner = find_runner(runners, "linux", image["arch"], False) entry = { "name": image["name"], @@ -120,6 +125,51 @@ def generate_docker_matrix_entries( return matrix_entries +def generate_crate_build_matrix_entries( + python_entries: list[dict[str, str]], + runners: dict[str, Any], + config: dict[str, Any], + force_crate_build: bool = False, + platform_filter: Optional[str] = None, +) -> list[dict[str, str]]: + """Generate matrix entries for crate builds based on python build matrix.""" + needed_builds = set() + for entry in python_entries: + # The crate build will need to match the runner's architecture + runner = runners[entry["runner"]] + needed_builds.add((entry["platform"], runner["arch"])) + + # If forcing crate build, also include all possible native builds + if force_crate_build: + for platform, platform_config in config.items(): + # Filter by platform if specified + if platform_filter and platform != platform_filter: + continue + + for target_config in platform_config.values(): + # Only include if native (run: true means native) + if not target_config.get("run"): + continue + + arch = target_config["arch"] + needed_builds.add((platform, arch)) + + # Create matrix entries for each needed build + return [ + { + "platform": platform, + "arch": arch, + "runner": find_runner(runners, platform, arch, True), + "crate_artifact_name": crate_artifact_name( + platform, + arch, + ), + } + for platform, arch in needed_builds + if not platform_filter or platform == platform_filter + ] + + def generate_python_build_matrix_entries( config: dict[str, Any], runners: dict[str, Any], @@ -154,10 +204,12 @@ def generate_python_build_matrix_entries( return matrix_entries -def find_runner(runners: dict[str, Any], platform: str, arch: str) -> str: +def find_runner(runners: dict[str, Any], platform: str, arch: str, free: bool) -> str: # Find a matching platform first match_platform = [ - runner for runner in runners if runners[runner]["platform"] == platform + runner + for runner in runners + if runners[runner]["platform"] == platform and runners[runner]["free"] == free ] # Then, find a matching architecture @@ -173,7 +225,9 @@ def find_runner(runners: dict[str, Any], platform: str, arch: str) -> str: if match_platform: return match_platform[0] - raise RuntimeError(f"No runner found for platform {platform!r} and arch {arch!r}") + raise RuntimeError( + f"No runner found for platform {platform!r} and arch {arch!r} with free={free}" + ) def add_python_build_entries_for_config( @@ -188,7 +242,7 @@ def add_python_build_entries_for_config( python_versions = config["python_versions"] build_options = config["build_options"] arch = config["arch"] - runner = find_runner(runners, platform, arch) + runner = find_runner(runners, platform, arch, False) # Create base entry that will be used for all variants base_entry = { @@ -199,6 +253,8 @@ def add_python_build_entries_for_config( # If `run` is in the config, use that — otherwise, default to if the # runner architecture matches the build architecture "run": str(config.get("run", runners[runner]["arch"] == arch)).lower(), + # Use the crate artifact built for the runner's architecture + "crate_artifact_name": crate_artifact_name(platform, runners[runner]["arch"]), } # Add optional fields if they exist @@ -266,9 +322,14 @@ def parse_args() -> argparse.Namespace: action="store_true", help="If only free runners should be used.", ) + parser.add_argument( + "--force-crate-build", + action="store_true", + help="Force crate builds to be included even without python builds.", + ) parser.add_argument( "--matrix-type", - choices=["python-build", "docker-build", "all"], + choices=["python-build", "docker-build", "crate-build", "all"], default="all", help="Which matrix types to generate (default: all)", ) @@ -295,16 +356,16 @@ def main() -> None: result = {} - # Generate python-build matrix if requested - python_entries = [] - if args.matrix_type in ["python-build", "all"]: - python_entries = generate_python_build_matrix_entries( - config, - runners, - args.platform, - labels, - ) + # Generate python build entries + python_entries = generate_python_build_matrix_entries( + config, + runners, + args.platform, + labels, + ) + # Output python-build matrix if requested + if args.matrix_type in ["python-build", "all"]: if args.max_shards: python_build_matrix = {} shards = (len(python_entries) // CI_MATRIX_SIZE_LIMIT) + 1 @@ -345,6 +406,17 @@ def main() -> None: ) result["docker-build"] = {"include": docker_entries} + # Generate crate-build matrix if requested + if args.matrix_type in ["crate-build", "all"]: + crate_entries = generate_crate_build_matrix_entries( + python_entries, + runners, + config, + args.force_crate_build, + args.platform, + ) + result["crate-build"] = {"include": crate_entries} + print(json.dumps(result))