feat(QA-200): Automated pythonindex package tests

.github/workflows/generate-index.yml

@@ -73,4 +73,18 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deploy
-        uses: actions/deploy-pages@v4
+        uses: actions/deploy-pages@v4
+
+  test:
+    strategy:
+      matrix:
+        python-version: ["3.13"]
+        wasmer-version: ["v6.1.0-rc.3"]
+    uses: ./.github/workflows/test-python-index.yaml
+    with:
+      python-version: ${{ matrix.python-version }}
+      wasmer-version: ${{ matrix.wasmer-version }}
+      # Test primary python index only for now. This
+      # setup allows for hosting a separate "testing" python
+      # index down the line (or simply a local one)
+      python-index: https://pythonindex.wasix.org

.github/workflows/test-python-index.yaml

@@ -0,0 +1,65 @@
+name: Test Workflow
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        required: true
+        type: string
+      wasmer-version:
+        required: true
+        type: string
+      python-index:
+        required: true
+        type: string
+      ubuntu-runner:
+        required: false
+        type: string
+        default: "ubuntu-24.04"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Setup environment
+        run: |
+          cd testing
+          sudo apt-get install libzbar0
+          pip install .
+      # Validate that the python tests themselves are valid
+      - name: Run tests natively
+        continue-on-error: true
+        run: |
+          cd testing
+          TEST_DIR=../tests/ python src/run-tests.py
+      # Secondly, validate that the tests pass within wasmer as well
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install wasmer
+        run: curl https://get.wasmer.io -sSfL | sh -s ${{ inputs.wasmer-version }}
+      - name: Setup project for wasmer
+        run: |
+          cd testing
+          uv pip compile pyproject.toml \
+            --python-version=${{ inputs.python-version }} \
+            --universal \
+            --extra-index-url ${{ inputs.python-index }}/simple \
+            --index-url=https://pypi.org/simple \
+            --emit-index-url \
+            --only-binary :all: \
+            -o wasmer-requirements.txt
+
+          uvx pip install -r wasmer-requirements.txt \
+            --python-version=${{ inputs.python-version }} \
+            --target wasix-site-packages \
+            --platform wasix_wasm32 \
+            --only-binary=:all: \
+            --compile
+      # Ensure tests pass on wasmer
+      - name: Run tests (wasmer)
+        run: cd testing && /home/runner/.wasmer/bin/wasmer run . --registry=wasmer.wtf --command-name=test --net

.gitignore

@@ -18,3 +18,5 @@ __pycache__
 /pkgs/*.build
 /pkgs/*.prepared
 /pkgs/*.sysroot
+wasix-site-packages
+wasmer-requirements.txt

.python-version

@@ -0,0 +1 @@
+3.13

README.md

@@ -260,6 +260,60 @@ psycopg3-c is just the sdist of psycopg3-binary
   * This is the commit that is currently used by wasix-libc
 <!-- LIB_VERSIONS_END -->
 
+## Tests
+
+The whole point of these tests is to ensure that the packages works within the wasmer runtime.
+So there are two ways of running the tests, natively and with wasmer:
+
+### Native
+
+Requirements:
+
+- Python 3.13
+- `python -m venv env`
+- `source venv/bin/activate`
+- `pip install .`
+- `pip install pytest`
+
+I'm sure it's possible to do via uv, or poetry, or many other ways. But this works also.
+
+Run tests:
+
+- `python -m pytest`
+- Discovers files matching `*-test.py`, `*_test.py`, and `test_*.py` under `tests/`.
+- Files ending with `.skip.py` are ignored.
+- Files ending with `-broken.py` are marked as expected failures (strict). If they pass, the run reports XPASS and fails.
+
+### Wasmer
+
+These are the instructions as of 2025-10, but there's a shipit looming, which may optimize the flow.
+
+Requirements:
+
+- wasmer 6.1.0-rc.3+
+
+Run tests:
+
+- `cd testing`
+- `uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt`
+- `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile`
+- `TEST_DIR=../tests/ wasmer run . --registry=wasmer.wtf --net --forward-host-env`
+- `curl localhost:8081/check`
+
+This will run all tests via fastapi.
+
+You may also run each test individually by:
+
+- `curl localhost:8081/list`
+- `curl localhost:8081/check/<test-file>`
+
+This is needed when testing on edge, since `.../check` times out the workload.
+In conjunction with this, there is a convenience script which runs all tests each in a separate query.
+
+So you may also run `./run-all-tests-via-api.py --host <hostname> --port <port>`.
+If you need to hit a specific IP while preserving the original hostname (e.g., for edge testing or custom DNS), use `--resolve-ip <ip>` which is SNI-compatible for HTTPS and sets the HTTP `Host` header accordingly.
+This is intended to be run to validate package functionaltiy on edge, as each test becomes a separate workload.
+
 ### Notes
 
 All built library packages should include a pkg-config file for each library.

app.yaml

@@ -0,0 +1,5 @@
+name: build-scripts
+app_id: da_nm3IetwU74x2
+owner: wasmer
+package: '.'
+kind: wasmer.io/App.v0

run-tests.sh

@@ -26,7 +26,7 @@ for testfile in tests/*.py; do
     fi
 
     EXPECT_BROKEN=false
-    if [[ "$TEST_NAME" == *-broken.py ]]; then
+    if [[ "$TEST_NAME" == *\.broken.py ]]; then
         EXPECT_BROKEN=true
     fi
 

testing/app.yaml

@@ -0,0 +1,5 @@
+kind: wasmer.io/App.v0
+name: build-scripts
+owner: lorentz-dev
+package: .
+app_id: da_2OPIqt7UeOLK

testing/pyproject.toml

@@ -0,0 +1,79 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "wasmer-build-scripts"
+version = "0.0.0"
+description = "Test scaffolding for build-scripts; runs tests via pytest."
+readme = "README.md"
+requires-python = ">=3.13"
+authors = [{ name = "Wasmer contributors" }]
+dependencies = [
+  "annotated-types",
+  "brotlicffi",
+  "certifi",
+  "cffi",
+  "charset-normalizer",
+  "cryptography",
+  "google-crc32c",
+  "fastapi>=0.116.1",
+  "idna",
+  "jiter",
+  "lxml",
+  "MarkupSafe",
+  "msgpack",
+  "mysqlclient",
+  "numpy",
+  "orjson",
+  "pandas",
+  "Pillow",
+  "protobuf",
+  "psycopg[binary]",
+  "pycparser",
+  "pycryptodome",
+  "pycryptodomex",
+  "pydantic>=2",
+  "PyNaCl",
+  "pyOpenSSL",
+  "pypandoc",
+  "pypng",
+  "pyzbar",
+  "qrcode",
+  "regex",
+  "requests",
+  "rpds-py",
+  "shapely",
+  "six",
+  "tiktoken",
+  "typing-inspection",
+  "typing_extensions",
+  "tzdata",
+  "urllib3",
+  "uvloop",
+  "uvicorn>=0.35.0",
+  "python-dateutil",
+  "psycopg",
+  "psycopg_pool",
+  "pytz",
+  "PyYAML",
+  "svgwrite",
+]
+
+[tool.setuptools.packages.find]
+include = []
+exclude = ["*"]
+
+
+[tool.pytest.ini_options]
+python_files = ["*-test.py", "*_test.py", "test_*.py"]
+addopts = ["-q"]
+testpaths = ["tests"]
+
+[[tool.uv.index]]
+# A human-friendly name you pick
+name = "wasix"
+# The Simple index URL
+url = "https://wasix-org.github.io/build-scripts/simple"
+# Optional – make this the primary index
+default = true