From 5635c1b7149ec9ef5d7aee8bf874c2b1f4d33e68 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Tue, 16 Sep 2025 19:44:57 +0300 Subject: [PATCH 01/33] feat: Setup local testing with pytest --- .gitignore | 1 + .python-version | 1 + README.md | 310 +++++++------- pyproject.toml | 73 ++++ run-tests.sh | 106 ----- tests/conftest.py | 21 + tests/mysqlclient-test.py | 15 +- tests/psycopg-pool-test.py | 18 +- tests/psycopg-test.py | 16 +- tests/requests-test.py | 17 +- tests/typing_extensions-test.py | 14 +- tests/urllib3-test.py | 16 +- tests/uvloop-test.py | 60 ++- wasmer-requirements.txt | 139 +++++++ wasmer.toml | 19 + wasmer_build_scripts.egg-info/PKG-INFO | 381 ++++++++++++++++++ wasmer_build_scripts.egg-info/SOURCES.txt | 8 + .../dependency_links.txt | 1 + wasmer_build_scripts.egg-info/requires.txt | 46 +++ wasmer_build_scripts.egg-info/top_level.txt | 1 + 20 files changed, 980 insertions(+), 283 deletions(-) create mode 100644 .python-version create mode 100644 pyproject.toml delete mode 100644 run-tests.sh create mode 100644 tests/conftest.py create mode 100644 wasmer-requirements.txt create mode 100644 wasmer.toml create mode 100644 wasmer_build_scripts.egg-info/PKG-INFO create mode 100644 wasmer_build_scripts.egg-info/SOURCES.txt create mode 100644 wasmer_build_scripts.egg-info/dependency_links.txt create mode 100644 wasmer_build_scripts.egg-info/requires.txt create mode 100644 wasmer_build_scripts.egg-info/top_level.txt diff --git a/.gitignore b/.gitignore index 8b4a5f4..07815c0 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ __pycache__ /pkgs/*.build /pkgs/*.prepared /pkgs/*.sysroot +wasix-site-packages diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/README.md b/README.md index b44775f..13f71ef 100644 --- a/README.md +++ b/README.md @@ -84,10 +84,10 @@ Buildscripts to build numpy and other wheels for wasix. For convenience, this pa The build script is controlled by the following environment variables: -* `CC`, `CXX`, `AR`, `LD`, `RANLIB`, etc... : The cross-compiler tools. These should all be normal clang tools, but target wasm32-wasix by default and use the wasix sysroot. -* `WASIX_SYSROOT`: The path to the wasix sysroot that is used by the toolchain. Libraries will get installed here when you run `make install` or when they are required to build a package. -* `INSTALL_DIR`: The path to the python library path. Wheels will get installed here when you run `make install`. -* `WASMER`: The path to the wasmer binary. You must have it registered to handle wasm files as binfmt_misc. You can do this with `sudo $WASMER binfmt reregister`. +- `CC`, `CXX`, `AR`, `LD`, `RANLIB`, etc... : The cross-compiler tools. These should all be normal clang tools, but target wasm32-wasix by default and use the wasix sysroot. +- `WASIX_SYSROOT`: The path to the wasix sysroot that is used by the toolchain. Libraries will get installed here when you run `make install` or when they are required to build a package. +- `INSTALL_DIR`: The path to the python library path. Wheels will get installed here when you run `make install`. +- `WASMER`: The path to the wasmer binary. You must have it registered to handle wasm files as binfmt_misc. You can do this with `sudo $WASMER binfmt reregister`. The easiest way to setup all the environment variables is to activate the wasix-clang environment using `source wasix-clang/activate`. @@ -154,111 +154,149 @@ Here is a list of the versions of the wheels and libraries that are included in #### Wheels -* numpy: numpy/numpy main -* markupsafe: 3.0.2 -* pandas: 2.3.2 -* pytz: 2025.2 -* dateutil: 2.9.0 -* tzdata: 2025.2 -* six: 1.17.0 -* msgpack: 1.1.0 -* pycryptodome: 3.23.0 -* pycryptodomex: 3.23.0 -* pyzbar: 0.1.9 -* cpython: 3.1.2 -* pypandoc: 1.15 -* pypandoc_binary: 1.15 -* psycopg: 3.2.9 -* psycopg-binary: 3.2.9 -* psycopg-pool: pool-3.2.6 -* brotlicffi: 1.1.0.0 -* cffi: 1.17.1 -* pillow: 11.3.0 -* matplotlib: 3.10.6 -* uvloop: 0.21.0 -* mysqlclient: 2.2.7 -* python-qrcode: 8.2 -* pycparser: 2.22 -* pydantic: 2.11.7 -* typing_extensions: 4.14.1 -* typing-inspection: 0.4.1 -* annotated-types: 0.7.0 -* shapely: 2.1.1 -* mrab-regex: 2025.7.31 -* lxml: 6.0.0 -* protobuf: 31.1 -* grpc: 1.74.1 -* numpy: 1.26.5 -* numpy: 2.0.2 -* numpy: 2.3.2 -* python-crc32c: 1.7.1 -* requests: 2.32.4 -* urllib3: 2.5.0 -* idna: 3.10 -* certifi: 2025.08.03 -* charset-normalizer: 3.4.3 -* pypng: 0.20250521.0 -* pyarrow: 19.0.1 -* pyarrow: 21.0.0 -* packaging: 25.0 -* pyparsing: 3.2.3 -* cycler: 0.12.1 -* kiwisolver: 1.4.9 -* contourpy: 1.3.3 -* pyopenssl: 25.1.0 -* aspw: 3.50.4.0 +- numpy: numpy/numpy main +- markupsafe: 3.0.2 +- pandas: 2.3.2 +- pytz: 2025.2 +- dateutil: 2.9.0 +- tzdata: 2025.2 +- six: 1.17.0 +- msgpack: 1.1.0 +- pycryptodome: 3.23.0 +- pycryptodomex: 3.23.0 +- pyzbar: 0.1.9 +- cpython: 3.1.2 +- pypandoc: 1.15 +- pypandoc_binary: 1.15 +- psycopg: 3.2.9 +- psycopg-binary: 3.2.9 +- psycopg-pool: pool-3.2.6 +- brotlicffi: 1.1.0.0 +- cffi: 1.17.1 +- pillow: 11.3.0 +- matplotlib: 3.10.6 +- uvloop: 0.21.0 +- mysqlclient: 2.2.7 +- python-qrcode: 8.2 +- pycparser: 2.22 +- pydantic: 2.11.7 +- typing_extensions: 4.14.1 +- typing-inspection: 0.4.1 +- annotated-types: 0.7.0 +- shapely: 2.1.1 +- mrab-regex: 2025.7.31 +- lxml: 6.0.0 +- protobuf: 31.1 +- grpc: 1.74.1 +- numpy: 1.26.5 +- numpy: 2.0.2 +- numpy: 2.3.2 +- python-crc32c: 1.7.1 +- requests: 2.32.4 +- urllib3: 2.5.0 +- idna: 3.10 +- certifi: 2025.08.03 +- charset-normalizer: 3.4.3 +- pypng: 0.20250521.0 +- pyarrow: 19.0.1 +- pyarrow: 21.0.0 +- packaging: 25.0 +- pyparsing: 3.2.3 +- cycler: 0.12.1 +- kiwisolver: 1.4.9 +- contourpy: 1.3.3 +- pyopenssl: 25.1.0 +- aspw: 3.50.4.0 -psycopg3-c is just the sdist of psycopg3-binary +psycopg3-c is just the sdist of psycopg3-binary## Running Tests #### Libraries -* libzbar: 0.23.93 -* libffi: wasix-org/libffi main -* pandoc: haskell-wasm/pandoc wasm -* postgresql: 17.5 -* brotli: 1.1.0 -* zlib: develop - * 1.3.1 does not have proper cmake support, so we are using develop for now -* libjpeg-turbo: 3.1.1 -* xz: 5.8.1 -* libtiff: 4.7.0 -* libwebp: 1.5.0 -* giflib: 5.2.2 -* libpng: 1.6.50 -* SDL: 3.2.16 - * SDL has all subsystems disabled -* openjpeg: 2.5.3 -* libuv: 1.51.0 -* mariadb-connector-c: 3.4.6 -* openssl: 3.5.1 -* bzip2: 1.0.8 -* util-linux: 2.41.1 - * We only build libuuid from util-linux -* openssh: 10.0p2 -* dropbear: 2025.88 -* tinyxml2: 11.0.0 -* geos: 3.13.1 -* libxslt: 1.1.43 -* libxml2: 2.14.5 -* google-crc32c: 1.1.2 -* arrow: 19.0.1 -* arrow: 21.0.0 -* rapidjson: 1.1.0 -* icu: 77.1 -* readline: 8.2 -* ncurses: 6.4.20230225 -* curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 - * This was the current commit in the wasix fork. We can probably update to the next release -* pycurl: 7.45.6 -* sqlite: 3.50.4 -* wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 - * The commit before we removed fork. A few packages don't build if fork is not defined. -* libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - * This is the commit that is currently used by wasix-libc -* compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - * This is the commit that is currently used by wasix-libc - +- libzbar: 0.23.93 +- libffi: wasix-org/libffi main +- pandoc: haskell-wasm/pandoc wasm +- postgresql: 17.5 +- brotli: 1.1.0 +- zlib: develop + - 1.3.1 does not have proper cmake support, so we are using develop for now +- libjpeg-turbo: 3.1.1 +- xz: 5.8.1 +- libtiff: 4.7.0 +- libwebp: 1.5.0 +- giflib: 5.2.2 +- libpng: 1.6.50 +- SDL: 3.2.16 + - SDL has all subsystems disabled +- openjpeg: 2.5.3 +- libuv: 1.51.0 +- mariadb-connector-c: 3.4.6 +- openssl: 3.5.1 +- bzip2: 1.0.8 +- util-linux: 2.41.1 + - We only build libuuid from util-linux +- openssh: 10.0p2 +- dropbear: 2025.88 +- tinyxml2: 11.0.0 +- geos: 3.13.1 +- libxslt: 1.1.43 +- libxml2: 2.14.5 +- google-crc32c: 1.1.2 +- arrow: 19.0.1 +- arrow: 21.0.0 +- rapidjson: 1.1.0 +- icu: 77.1 +- readline: 8.2 +- ncurses: 6.4.20230225 +- curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 + - This was the current commit in the wasix fork. We can probably update to the next release +- pycurl: 7.45.6 +- sqlite: 3.50.4 +- wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 + - The commit before we removed fork. A few packages don't build if fork is not defined. +- libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + - This is the commit that is currently used by wasix-libc +- compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + - This is the commit that is currently used by wasix-libc + + +## Tests + +The whole point of these tests is to ensure that the packages works within the wasmer runtime. +So there are two ways of running the tests, natively and with wasmer: + +### Native + +Requirements: + +- Python 3.10+ +- `python -m venv env` +- `source venv/bin/activate` +- `pip install .` +- `pip install pytest` + +I'm sure it's possible to do via uv, or poetry, or many other ways. But this works also. + +Run tests: + +- `python -m pytest` +- Discovers files matching `*-test.py`, `*_test.py`, and `test_*.py` under `tests/`. +- Files ending with `.skip.py` are ignored. +- Files ending with `-broken.py` are marked as expected failures (strict). If they pass, the run reports XPASS and fails. + +### Wasmer + +These are the instructions as of 2025-10, but there's a shipit looming, which may optimize the flow. +If this doesn't work though, we have a bug: + +Requirements: + +- wasmer 6.1.0-rc.3+ + +Run tests: + +- `uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt` +- `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile` ### Notes @@ -286,7 +324,7 @@ to check which python libraries depend on shared libs. We try to keep that to a ### Structure - + +### Notes + +All built library packages should include a pkg-config file for each library. + +When building wheels that depend on detecting numpy headers via `numpy.get_include()` at compiletime it might be required to set the `NUMPY_ONLY_GET_INCLUDE` environment variable. We have a special patch that detects that variable when importing numpy and only exports the `get_include()` function in that case. Otherwise importing numpy causes it to load all native modules and crash when crosscompiling because wasm native modules won't work on x86_64 CPUs. + +While `wasix-clang` tries to be as lightweight as possible while still behaving like clang, we have a special `WASIX_FORCE_STATIC_DEPENDENCIES` environment variable that forces all libraries to be static. While WASIX does have full support we don't always like to use shared libs for reasons. + +#### [Variables in pkg-config files](https://www.gnu.org/prep/standards/html_node/Directory-Variables.html) + +When building libs, we should make shour they include pkg-config files. The pkg config files should have their prefix set to `/usr/local`, exec_prefix set to `${prefix}`, libdir set to `${exec_prefix}/lib/wasm32-wasi`, and includedir set to `${prefix}/include`. In some cases it might be acceptable to have exec_prefix hardcoded to the same value as prefix. In that case libdir should start with `${prefix}` instead of `${exec_prefix}`. + +### Analyzing the output + +#### Check whether python libaries require shared libs + +You can run somethign like + +```bash +for f in $(find "$INSTALL_DIR" -name '*.so') ; do echo $f ; wasm-tools print $f | head -n10 | grep '(needed' -C10 ; done +``` + +to check which python libraries depend on shared libs. We try to keep that to a minimum, so wheels contain everything that is required to use a package. + +### Structure + + + +Inside the pkgs/ folder there can be the following directories: + +- `*.source`: clean submodule checkout +- `*.prepared`: patched worktree of source +- `*.build`: temporary build directory +- `*.tar.gz`: python sdist +- `*.sdist`: unpacked python sdist +- `*.whl`: compiled python wheel +- `*.wheel`: unpacked python wheel +- `*.lib`: unpacked library/application +- `*.tar.xz`: packed library/application + +#### Base structure + +Each project follows a consistent flow through the first three main directories. + +- `*.source` + - This is a clean checkout of the project's upstream source code, tracked as a git submodule. + - We avoid modifying this directly, since changes here would slow down git operations in the build-scripts repo. +- `*.prepared` + - A git worktree created from the `*.source` repository. + - If patches are needed, they're applied here. + - If no patches are needed, it's just a clean mirror of the source. + - This directory is persistent and only refreshed if the source changes so new patches can be developed in this directory +- `*.build` + - A copy of the `*.prepared` directory, used for the actual build step. + - Contains all intermediate build artifacts. + - This directory is temporary and may be deleted between builds. Never make manual changes here. + +The remaining steps are different depending on the type of project. + +#### Python modules + +- The build step creates a `*.tar.gz` sdist from the `*.build` directory. +- The sdist is then extracted into a `*.sdist` folder. +- Finally, a wheel (`*.whl`) is built from the `*.sdist`. +- If you want to you can make a `*.wheel` directory to view the unpacked wheel + +#### WASIX libraries and applications + +- The build step builds the library and installs it into a `*.lib` folder, following the correct directory structure. +- That folder is then compressed into a final distributable \*.tar.xz. + +#### Interdependencies + +If a project depends on other project they can either be direct dependencies of that project or you can define a `*.sysroot` target with the dependencies as prerequisites. + +- `*.sysroot` + - Contains the merged builds of multiple other projects + - Useful when a project is using pkg-config to find its dependencies + - Automatically builds a sysroot from its list of prerequisites diff --git a/wasmer_build_scripts.egg-info/SOURCES.txt b/wasmer_build_scripts.egg-info/SOURCES.txt new file mode 100644 index 0000000..8c1bd93 --- /dev/null +++ b/wasmer_build_scripts.egg-info/SOURCES.txt @@ -0,0 +1,8 @@ +LICENSE +README.md +pyproject.toml +wasmer_build_scripts.egg-info/PKG-INFO +wasmer_build_scripts.egg-info/SOURCES.txt +wasmer_build_scripts.egg-info/dependency_links.txt +wasmer_build_scripts.egg-info/requires.txt +wasmer_build_scripts.egg-info/top_level.txt \ No newline at end of file diff --git a/wasmer_build_scripts.egg-info/dependency_links.txt b/wasmer_build_scripts.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/wasmer_build_scripts.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/wasmer_build_scripts.egg-info/requires.txt b/wasmer_build_scripts.egg-info/requires.txt new file mode 100644 index 0000000..95137a4 --- /dev/null +++ b/wasmer_build_scripts.egg-info/requires.txt @@ -0,0 +1,46 @@ +annotated-types +brotlicffi +certifi +cffi +charset-normalizer +cryptography +google-crc32c +idna +jiter +lxml +MarkupSafe +msgpack +mysqlclient +numpy +orjson +pandas +Pillow +protobuf +psycopg[binary] +pycparser +pycryptodome +pycryptodomex +pydantic>=2 +PyNaCl +pyOpenSSL +pypandoc +pypng +pyzbar +qrcode +regex +requests +rpds-py +shapely +six +tiktoken +typing-inspection +typing_extensions +tzdata +urllib3 +uvloop +python-dateutil +psycopg +psycopg_pool +pytz +PyYAML +svgwrite diff --git a/wasmer_build_scripts.egg-info/top_level.txt b/wasmer_build_scripts.egg-info/top_level.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/wasmer_build_scripts.egg-info/top_level.txt @@ -0,0 +1 @@ + From 26265fb49acc97e85d769de48fe377fa29d58808 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Wed, 17 Sep 2025 08:14:46 +0300 Subject: [PATCH 02/33] feat: Added support for running tests in wasmer runtime --- pyproject.toml | 1 + wasmer-requirements.txt | 17 ++++++++++++++++- wasmer.toml | 4 ++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4d4d227..6cb1fa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ dependencies = [ "python-dateutil", "psycopg", "psycopg_pool", + "pytest", "pytz", "PyYAML", "svgwrite", diff --git a/wasmer-requirements.txt b/wasmer-requirements.txt index 441ae36..16c756f 100644 --- a/wasmer-requirements.txt +++ b/wasmer-requirements.txt @@ -24,7 +24,9 @@ charset-normalizer==3.4.3 # wasmer-build-scripts (pyproject.toml) # requests colorama==0.4.6 ; sys_platform == 'win32' - # via qrcode + # via + # pytest + # qrcode cryptography==45.0.4 # via # wasmer-build-scripts (pyproject.toml) @@ -35,6 +37,8 @@ idna==3.10 # via # wasmer-build-scripts (pyproject.toml) # requests +iniconfig==2.1.0 + # via pytest jiter==0.10.0 # via wasmer-build-scripts (pyproject.toml) lxml==6.0.0 @@ -52,16 +56,22 @@ numpy==2.3.2 # shapely orjson==3.11.0 # via wasmer-build-scripts (pyproject.toml) +packaging==25.0 + # via pytest pandas==2.3.2 # via wasmer-build-scripts (pyproject.toml) pillow==11.3.0 # via wasmer-build-scripts (pyproject.toml) +pluggy==1.6.0 + # via pytest protobuf==6.31.1 # via wasmer-build-scripts (pyproject.toml) psycopg==3.2.9 # via wasmer-build-scripts (pyproject.toml) psycopg-binary==3.2.9 ; implementation_name != 'pypy' # via psycopg +psycopg-pool==3.2.6 + # via wasmer-build-scripts (pyproject.toml) pycparser==2.22 # via # wasmer-build-scripts (pyproject.toml) @@ -74,6 +84,8 @@ pydantic==2.11.7 # via wasmer-build-scripts (pyproject.toml) pydantic-core==2.33.2 # via pydantic +pygments==2.19.2 + # via pytest pynacl==1.5.0 # via wasmer-build-scripts (pyproject.toml) pyopenssl==25.1.0 @@ -82,6 +94,8 @@ pypandoc==1.15 # via wasmer-build-scripts (pyproject.toml) pypng==0.20250521.0 # via wasmer-build-scripts (pyproject.toml) +pytest==8.4.2 + # via wasmer-build-scripts (pyproject.toml) python-dateutil==2.9.0 # via # wasmer-build-scripts (pyproject.toml) @@ -119,6 +133,7 @@ tiktoken==0.9.0 typing-extensions==4.14.1 # via # wasmer-build-scripts (pyproject.toml) + # psycopg-pool # pydantic # pydantic-core # typing-inspection diff --git a/wasmer.toml b/wasmer.toml index 0fe181e..a39bff8 100644 --- a/wasmer.toml +++ b/wasmer.toml @@ -6,7 +6,7 @@ version = "0.1.1" "wasmer/python-native" = "=0.1.3" [fs] -"/app" = "./src" +"/tests" = "./tests" "/cpython/lib/python3.13/site-packages" = "./wasix-site-packages" [[command]] @@ -15,5 +15,5 @@ module = "wasmer/python-native:python" runner = "wasi" [command.annotations.wasi] -main-args = ["/app/main.py"] +main-args = ["-m", "pytest"] env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "HOME=/app"] From a943e0cfb6d88e3ebb44bb8b88d0feb5f2f75925 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Wed, 17 Sep 2025 08:22:03 +0300 Subject: [PATCH 03/33] docs: Changed back formatting to not joink git blame --- README.md | 294 +++++++++++++++++++++++++++--------------------------- 1 file changed, 147 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index 13f71ef..2c32802 100644 --- a/README.md +++ b/README.md @@ -84,10 +84,10 @@ Buildscripts to build numpy and other wheels for wasix. For convenience, this pa The build script is controlled by the following environment variables: -- `CC`, `CXX`, `AR`, `LD`, `RANLIB`, etc... : The cross-compiler tools. These should all be normal clang tools, but target wasm32-wasix by default and use the wasix sysroot. -- `WASIX_SYSROOT`: The path to the wasix sysroot that is used by the toolchain. Libraries will get installed here when you run `make install` or when they are required to build a package. -- `INSTALL_DIR`: The path to the python library path. Wheels will get installed here when you run `make install`. -- `WASMER`: The path to the wasmer binary. You must have it registered to handle wasm files as binfmt_misc. You can do this with `sudo $WASMER binfmt reregister`. +* `CC`, `CXX`, `AR`, `LD`, `RANLIB`, etc... : The cross-compiler tools. These should all be normal clang tools, but target wasm32-wasix by default and use the wasix sysroot. +* `WASIX_SYSROOT`: The path to the wasix sysroot that is used by the toolchain. Libraries will get installed here when you run `make install` or when they are required to build a package. +* `INSTALL_DIR`: The path to the python library path. Wheels will get installed here when you run `make install`. +* `WASMER`: The path to the wasmer binary. You must have it registered to handle wasm files as binfmt_misc. You can do this with `sudo $WASMER binfmt reregister`. The easiest way to setup all the environment variables is to activate the wasix-clang environment using `source wasix-clang/activate`. @@ -154,111 +154,111 @@ Here is a list of the versions of the wheels and libraries that are included in #### Wheels -- numpy: numpy/numpy main -- markupsafe: 3.0.2 -- pandas: 2.3.2 -- pytz: 2025.2 -- dateutil: 2.9.0 -- tzdata: 2025.2 -- six: 1.17.0 -- msgpack: 1.1.0 -- pycryptodome: 3.23.0 -- pycryptodomex: 3.23.0 -- pyzbar: 0.1.9 -- cpython: 3.1.2 -- pypandoc: 1.15 -- pypandoc_binary: 1.15 -- psycopg: 3.2.9 -- psycopg-binary: 3.2.9 -- psycopg-pool: pool-3.2.6 -- brotlicffi: 1.1.0.0 -- cffi: 1.17.1 -- pillow: 11.3.0 -- matplotlib: 3.10.6 -- uvloop: 0.21.0 -- mysqlclient: 2.2.7 -- python-qrcode: 8.2 -- pycparser: 2.22 -- pydantic: 2.11.7 -- typing_extensions: 4.14.1 -- typing-inspection: 0.4.1 -- annotated-types: 0.7.0 -- shapely: 2.1.1 -- mrab-regex: 2025.7.31 -- lxml: 6.0.0 -- protobuf: 31.1 -- grpc: 1.74.1 -- numpy: 1.26.5 -- numpy: 2.0.2 -- numpy: 2.3.2 -- python-crc32c: 1.7.1 -- requests: 2.32.4 -- urllib3: 2.5.0 -- idna: 3.10 -- certifi: 2025.08.03 -- charset-normalizer: 3.4.3 -- pypng: 0.20250521.0 -- pyarrow: 19.0.1 -- pyarrow: 21.0.0 -- packaging: 25.0 -- pyparsing: 3.2.3 -- cycler: 0.12.1 -- kiwisolver: 1.4.9 -- contourpy: 1.3.3 -- pyopenssl: 25.1.0 -- aspw: 3.50.4.0 +* numpy: numpy/numpy main +* markupsafe: 3.0.2 +* pandas: 2.3.2 +* pytz: 2025.2 +* dateutil: 2.9.0 +* tzdata: 2025.2 +* six: 1.17.0 +* msgpack: 1.1.0 +* pycryptodome: 3.23.0 +* pycryptodomex: 3.23.0 +* pyzbar: 0.1.9 +* cpython: 3.1.2 +* pypandoc: 1.15 +* pypandoc_binary: 1.15 +* psycopg: 3.2.9 +* psycopg-binary: 3.2.9 +* psycopg-pool: pool-3.2.6 +* brotlicffi: 1.1.0.0 +* cffi: 1.17.1 +* pillow: 11.3.0 +* matplotlib: 3.10.6 +* uvloop: 0.21.0 +* mysqlclient: 2.2.7 +* python-qrcode: 8.2 +* pycparser: 2.22 +* pydantic: 2.11.7 +* typing_extensions: 4.14.1 +* typing-inspection: 0.4.1 +* annotated-types: 0.7.0 +* shapely: 2.1.1 +* mrab-regex: 2025.7.31 +* lxml: 6.0.0 +* protobuf: 31.1 +* grpc: 1.74.1 +* numpy: 1.26.5 +* numpy: 2.0.2 +* numpy: 2.3.2 +* python-crc32c: 1.7.1 +* requests: 2.32.4 +* urllib3: 2.5.0 +* idna: 3.10 +* certifi: 2025.08.03 +* charset-normalizer: 3.4.3 +* pypng: 0.20250521.0 +* pyarrow: 19.0.1 +* pyarrow: 21.0.0 +* packaging: 25.0 +* pyparsing: 3.2.3 +* cycler: 0.12.1 +* kiwisolver: 1.4.9 +* contourpy: 1.3.3 +* pyopenssl: 25.1.0 +* aspw: 3.50.4.0 -psycopg3-c is just the sdist of psycopg3-binary## Running Tests +psycopg3-c is just the sdist of psycopg3-binary #### Libraries -- libzbar: 0.23.93 -- libffi: wasix-org/libffi main -- pandoc: haskell-wasm/pandoc wasm -- postgresql: 17.5 -- brotli: 1.1.0 -- zlib: develop - - 1.3.1 does not have proper cmake support, so we are using develop for now -- libjpeg-turbo: 3.1.1 -- xz: 5.8.1 -- libtiff: 4.7.0 -- libwebp: 1.5.0 -- giflib: 5.2.2 -- libpng: 1.6.50 -- SDL: 3.2.16 - - SDL has all subsystems disabled -- openjpeg: 2.5.3 -- libuv: 1.51.0 -- mariadb-connector-c: 3.4.6 -- openssl: 3.5.1 -- bzip2: 1.0.8 -- util-linux: 2.41.1 - - We only build libuuid from util-linux -- openssh: 10.0p2 -- dropbear: 2025.88 -- tinyxml2: 11.0.0 -- geos: 3.13.1 -- libxslt: 1.1.43 -- libxml2: 2.14.5 -- google-crc32c: 1.1.2 -- arrow: 19.0.1 -- arrow: 21.0.0 -- rapidjson: 1.1.0 -- icu: 77.1 -- readline: 8.2 -- ncurses: 6.4.20230225 -- curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 - - This was the current commit in the wasix fork. We can probably update to the next release -- pycurl: 7.45.6 -- sqlite: 3.50.4 -- wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 - - The commit before we removed fork. A few packages don't build if fork is not defined. -- libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - - This is the commit that is currently used by wasix-libc -- compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - - This is the commit that is currently used by wasix-libc - +* libzbar: 0.23.93 +* libffi: wasix-org/libffi main +* pandoc: haskell-wasm/pandoc wasm +* postgresql: 17.5 +* brotli: 1.1.0 +* zlib: develop + * 1.3.1 does not have proper cmake support, so we are using develop for now +* libjpeg-turbo: 3.1.1 +* xz: 5.8.1 +* libtiff: 4.7.0 +* libwebp: 1.5.0 +* giflib: 5.2.2 +* libpng: 1.6.50 +* SDL: 3.2.16 + * SDL has all subsystems disabled +* openjpeg: 2.5.3 +* libuv: 1.51.0 +* mariadb-connector-c: 3.4.6 +* openssl: 3.5.1 +* bzip2: 1.0.8 +* util-linux: 2.41.1 + * We only build libuuid from util-linux +* openssh: 10.0p2 +* dropbear: 2025.88 +* tinyxml2: 11.0.0 +* geos: 3.13.1 +* libxslt: 1.1.43 +* libxml2: 2.14.5 +* google-crc32c: 1.1.2 +* arrow: 19.0.1 +* arrow: 21.0.0 +* rapidjson: 1.1.0 +* icu: 77.1 +* readline: 8.2 +* ncurses: 6.4.20230225 +* curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 + * This was the current commit in the wasix fork. We can probably update to the next release +* pycurl: 7.45.6 +* sqlite: 3.50.4 +* wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 + * The commit before we removed fork. A few packages don't build if fork is not defined. +* libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + * This is the commit that is currently used by wasix-libc +* compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + * This is the commit that is currently used by wasix-libc + ## Tests @@ -269,20 +269,20 @@ So there are two ways of running the tests, natively and with wasmer: Requirements: -- Python 3.10+ -- `python -m venv env` -- `source venv/bin/activate` -- `pip install .` -- `pip install pytest` +* Python 3.10+ +* `python -m venv env` +* `source venv/bin/activate` +* `pip install .` +* `pip install pytest` I'm sure it's possible to do via uv, or poetry, or many other ways. But this works also. Run tests: -- `python -m pytest` -- Discovers files matching `*-test.py`, `*_test.py`, and `test_*.py` under `tests/`. -- Files ending with `.skip.py` are ignored. -- Files ending with `-broken.py` are marked as expected failures (strict). If they pass, the run reports XPASS and fails. +* `python -m pytest` +* Discovers files matching `*-test.py`, `*_test.py`, and `test_*.py` under `tests/`. +* Files ending with `.skip.py` are ignored. +* Files ending with `-broken.py` are marked as expected failures (strict). If they pass, the run reports XPASS and fails. ### Wasmer @@ -291,12 +291,12 @@ If this doesn't work though, we have a bug: Requirements: -- wasmer 6.1.0-rc.3+ +* wasmer 6.1.0-rc.3+ Run tests: - `uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt` -- `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile` +* `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile` ### Notes @@ -324,7 +324,7 @@ to check which python libraries depend on shared libs. We try to keep that to a ### Structure - - -### Notes - -All built library packages should include a pkg-config file for each library. - -When building wheels that depend on detecting numpy headers via `numpy.get_include()` at compiletime it might be required to set the `NUMPY_ONLY_GET_INCLUDE` environment variable. We have a special patch that detects that variable when importing numpy and only exports the `get_include()` function in that case. Otherwise importing numpy causes it to load all native modules and crash when crosscompiling because wasm native modules won't work on x86_64 CPUs. - -While `wasix-clang` tries to be as lightweight as possible while still behaving like clang, we have a special `WASIX_FORCE_STATIC_DEPENDENCIES` environment variable that forces all libraries to be static. While WASIX does have full support we don't always like to use shared libs for reasons. - -#### [Variables in pkg-config files](https://www.gnu.org/prep/standards/html_node/Directory-Variables.html) - -When building libs, we should make shour they include pkg-config files. The pkg config files should have their prefix set to `/usr/local`, exec_prefix set to `${prefix}`, libdir set to `${exec_prefix}/lib/wasm32-wasi`, and includedir set to `${prefix}/include`. In some cases it might be acceptable to have exec_prefix hardcoded to the same value as prefix. In that case libdir should start with `${prefix}` instead of `${exec_prefix}`. - -### Analyzing the output - -#### Check whether python libaries require shared libs - -You can run somethign like - -```bash -for f in $(find "$INSTALL_DIR" -name '*.so') ; do echo $f ; wasm-tools print $f | head -n10 | grep '(needed' -C10 ; done -``` - -to check which python libraries depend on shared libs. We try to keep that to a minimum, so wheels contain everything that is required to use a package. - -### Structure - - - -Inside the pkgs/ folder there can be the following directories: - -- `*.source`: clean submodule checkout -- `*.prepared`: patched worktree of source -- `*.build`: temporary build directory -- `*.tar.gz`: python sdist -- `*.sdist`: unpacked python sdist -- `*.whl`: compiled python wheel -- `*.wheel`: unpacked python wheel -- `*.lib`: unpacked library/application -- `*.tar.xz`: packed library/application - -#### Base structure - -Each project follows a consistent flow through the first three main directories. - -- `*.source` - - This is a clean checkout of the project's upstream source code, tracked as a git submodule. - - We avoid modifying this directly, since changes here would slow down git operations in the build-scripts repo. -- `*.prepared` - - A git worktree created from the `*.source` repository. - - If patches are needed, they're applied here. - - If no patches are needed, it's just a clean mirror of the source. - - This directory is persistent and only refreshed if the source changes so new patches can be developed in this directory -- `*.build` - - A copy of the `*.prepared` directory, used for the actual build step. - - Contains all intermediate build artifacts. - - This directory is temporary and may be deleted between builds. Never make manual changes here. - -The remaining steps are different depending on the type of project. - -#### Python modules - -- The build step creates a `*.tar.gz` sdist from the `*.build` directory. -- The sdist is then extracted into a `*.sdist` folder. -- Finally, a wheel (`*.whl`) is built from the `*.sdist`. -- If you want to you can make a `*.wheel` directory to view the unpacked wheel - -#### WASIX libraries and applications - -- The build step builds the library and installs it into a `*.lib` folder, following the correct directory structure. -- That folder is then compressed into a final distributable \*.tar.xz. - -#### Interdependencies - -If a project depends on other project they can either be direct dependencies of that project or you can define a `*.sysroot` target with the dependencies as prerequisites. - -- `*.sysroot` - - Contains the merged builds of multiple other projects - - Useful when a project is using pkg-config to find its dependencies - - Automatically builds a sysroot from its list of prerequisites diff --git a/wasmer_build_scripts.egg-info/SOURCES.txt b/wasmer_build_scripts.egg-info/SOURCES.txt deleted file mode 100644 index 8c1bd93..0000000 --- a/wasmer_build_scripts.egg-info/SOURCES.txt +++ /dev/null @@ -1,8 +0,0 @@ -LICENSE -README.md -pyproject.toml -wasmer_build_scripts.egg-info/PKG-INFO -wasmer_build_scripts.egg-info/SOURCES.txt -wasmer_build_scripts.egg-info/dependency_links.txt -wasmer_build_scripts.egg-info/requires.txt -wasmer_build_scripts.egg-info/top_level.txt \ No newline at end of file diff --git a/wasmer_build_scripts.egg-info/dependency_links.txt b/wasmer_build_scripts.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/wasmer_build_scripts.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/wasmer_build_scripts.egg-info/requires.txt b/wasmer_build_scripts.egg-info/requires.txt deleted file mode 100644 index 95137a4..0000000 --- a/wasmer_build_scripts.egg-info/requires.txt +++ /dev/null @@ -1,46 +0,0 @@ -annotated-types -brotlicffi -certifi -cffi -charset-normalizer -cryptography -google-crc32c -idna -jiter -lxml -MarkupSafe -msgpack -mysqlclient -numpy -orjson -pandas -Pillow -protobuf -psycopg[binary] -pycparser -pycryptodome -pycryptodomex -pydantic>=2 -PyNaCl -pyOpenSSL -pypandoc -pypng -pyzbar -qrcode -regex -requests -rpds-py -shapely -six -tiktoken -typing-inspection -typing_extensions -tzdata -urllib3 -uvloop -python-dateutil -psycopg -psycopg_pool -pytz -PyYAML -svgwrite diff --git a/wasmer_build_scripts.egg-info/top_level.txt b/wasmer_build_scripts.egg-info/top_level.txt deleted file mode 100644 index 8b13789..0000000 --- a/wasmer_build_scripts.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ - From 96e26b69c9979947c0393ca49a3f80c1d1768a5e Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Wed, 17 Sep 2025 18:17:42 +0300 Subject: [PATCH 12/33] tweaks: Inject psql server details as env vars, spelling --- run-tests.py | 2 +- tests/psycopg-pool-test.py | 8 +++++++- tests/psycopg-test.py | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/run-tests.py b/run-tests.py index ecb6abf..78b1ab3 100755 --- a/run-tests.py +++ b/run-tests.py @@ -16,7 +16,7 @@ else: test_dir = os.getenv("TEST_DIR", "./tests/") g = f"{test_dir}*.py" - print(f"CHecking glob: {g}") + print(f"Checking glob: {g}") # Find all Python test files in ./tests directory TEST_FILES = [f for f in glob.glob(g) if ".skip" not in f and ".broken" not in f] diff --git a/tests/psycopg-pool-test.py b/tests/psycopg-pool-test.py index 5703e10..6f91fde 100644 --- a/tests/psycopg-pool-test.py +++ b/tests/psycopg-pool-test.py @@ -1,10 +1,16 @@ # docker run --rm -it --name some-postgres -e POSTGRES_USER=myuser -e POSTGRES_PASSWORD=mypassword -e POSTGRES_DB=mydatabase -p 5432:5432 postgres +import os + from psycopg_pool import ConnectionPool try: pool = ConnectionPool( - "dbname=mydatabase user=myuser password=mypassword host=localhost port=5432" + dbname=os.environ.get("POSTGRES_DB", "docker-local"), + user=os.environ.get("POSTGRES_USER", "postgres"), + password=os.environ.get("POSTGRES_PASSWORD", "securesecret"), + host=os.environ.get("DB_HOST", "0.0.0.0"), + port=os.environ.get("DB_PORT", "5432"), ) with pool.connection() as conn: with conn.cursor() as cur: diff --git a/tests/psycopg-test.py b/tests/psycopg-test.py index aa51ebf..cc1fb72 100644 --- a/tests/psycopg-test.py +++ b/tests/psycopg-test.py @@ -1,10 +1,16 @@ # docker run --rm -it --name some-postgres -e POSTGRES_USER=myuser -e POSTGRES_PASSWORD=mypassword -e POSTGRES_DB=mydatabase -p 5432:5432 postgres +import os + import psycopg try: conn = psycopg.connect( - "dbname=mydatabase user=myuser password=mypassword host=localhost port=5432" + dbname=os.environ.get("POSTGRES_DB", "docker-local"), + user=os.environ.get("POSTGRES_USER", "postgres"), + password=os.environ.get("POSTGRES_PASSWORD", "securesecret"), + host=os.environ.get("DB_HOST", "0.0.0.0"), + port=os.environ.get("DB_PORT", "5432"), ) cur = conn.cursor() cur.execute("SELECT version();") From 2ea54f23e026ec6e11439bc09722cb5c9ff7fc0c Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 07:57:03 +0300 Subject: [PATCH 13/33] tweaks: Removed pytest from dependencies --- pyproject.toml | 1 - wasmer.toml | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6cb1fa0..4d4d227 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ dependencies = [ "python-dateutil", "psycopg", "psycopg_pool", - "pytest", "pytz", "PyYAML", "svgwrite", diff --git a/wasmer.toml b/wasmer.toml index c0afae8..217c744 100644 --- a/wasmer.toml +++ b/wasmer.toml @@ -16,4 +16,9 @@ runner = "wasi" [command.annotations.wasi] main-args = ["/app/run-tests.py"] -env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "HOME=/app"] +env = [ + "PYTHONEXECUTABLE=/bin/python", + "PYTHONHOME=/cpython", + "PYTHONPATH=/opt/venv/lib/python3.13/site-packages", + "HOME=/app", +] From bcdf4a33f80eda89c3da6d2b935ff11bece852f5 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 09:16:46 +0300 Subject: [PATCH 14/33] feat: Converted project to be fastapi This way we can call the checks whenever we want. And hopefully shipit can do whatever magic it does to make it #justwork when deployed to edge --- app.yaml | 5 ++ main.py | 80 +++++++++++++++++++ pyproject.toml | 2 + run-tests.py | 14 +++- ...ol-test.py => psycopg-pool-test.broken.py} | 0 ...psycopg-test.py => psycopg-test.broken.py} | 0 wasmer-requirements.txt | 32 +++++--- wasmer.toml | 11 +-- 8 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 app.yaml create mode 100644 main.py rename tests/{psycopg-pool-test.py => psycopg-pool-test.broken.py} (100%) rename tests/{psycopg-test.py => psycopg-test.broken.py} (100%) diff --git a/app.yaml b/app.yaml new file mode 100644 index 0000000..8e91d1c --- /dev/null +++ b/app.yaml @@ -0,0 +1,5 @@ +name: build-scripts +app_id: da_nm3IetwU74x2 +owner: wasmer +package: '.' +kind: wasmer.io/App.v0 diff --git a/main.py b/main.py new file mode 100644 index 0000000..0a062ba --- /dev/null +++ b/main.py @@ -0,0 +1,80 @@ +import contextlib +import io +import os +import runpy +import sys + +from fastapi import FastAPI, Response + + +class Tee(io.StringIO): + def __init__(self, *streams): + super().__init__() + self.streams = streams + + def write(self, data): + for s in self.streams: + s.write(data) + s.flush() + super().write(data) + + +app = FastAPI( + title="Build Scripts - Tests", + description="This API wraps the existing run-scripts.py and simply calls it. This allows us to ensure that all tested packages works at any given time.", + version="0.1.0", +) + + +@app.get("/") +async def root(): + return { + "message": "Build Scripts - Tests", + "version": "0.1.0", + "endpoints": [ + "/", + "/check", + ], + } + + +import traceback + + +@app.get("/check") +async def check_packages(): + """ + Check packages by running the run-tests.py file and capture output. + It's done this way to allow the run-tests.py to also be run as + standalone script (and since I don't have time to refactor this + into a fully-fleged fastapi project) + """ + buf = Tee(sys.stdout) + try: + with contextlib.redirect_stdout(buf): + runpy.run_path("/app/run-tests.py") + output = buf.getvalue() + return Response(content=output, status_code=200) + except SystemExit as e: + output = buf.getvalue() + print(output) + if e.code == 0: + return Response(content=output, status_code=200) + else: + # Return status code 417 "Expectation failed", which is probably best fit + # even though we dont process any headers + return Response(content=output, status_code=417) + except Exception: + output = buf.getvalue() + tb = traceback.format_exc() + print(output) + print(tb) + return Response(content=output + "\n" + tb, status_code=500) + + +if __name__ == "__main__": + import uvicorn + + host = os.environ.get("HOST", "0.0.0.0") + port = int(os.environ.get("PORT", 8081)) + uvicorn.run(app, host=host, port=port) diff --git a/pyproject.toml b/pyproject.toml index 4d4d227..eb09f91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "charset-normalizer", "cryptography", "google-crc32c", + "fastapi>=0.116.1", "idna", "jiter", "lxml", @@ -46,6 +47,7 @@ dependencies = [ "tzdata", "urllib3", "uvloop", + "uvicorn>=0.35.0", "python-dateutil", "psycopg", "psycopg_pool", diff --git a/run-tests.py b/run-tests.py index 78b1ab3..0e28acf 100755 --- a/run-tests.py +++ b/run-tests.py @@ -5,6 +5,18 @@ import runpy import sys +class Tee(io.StringIO): + def __init__(self, *streams): + super().__init__() + self.streams = streams + + def write(self, data): + for s in self.streams: + s.write(data) + s.flush() + super().write(data) + + # Check if a specific test is provided if len(sys.argv) > 1: TEST_FILE = sys.argv[1] @@ -31,7 +43,7 @@ def run_test(test_file): print("{}Running test: {}{}".format(YELLOW, test_file, NC)) - buf = io.StringIO() + buf = Tee(sys.stdout) try: with contextlib.redirect_stdout(buf): runpy.run_path(test_file, run_name="__main__") diff --git a/tests/psycopg-pool-test.py b/tests/psycopg-pool-test.broken.py similarity index 100% rename from tests/psycopg-pool-test.py rename to tests/psycopg-pool-test.broken.py diff --git a/tests/psycopg-test.py b/tests/psycopg-test.broken.py similarity index 100% rename from tests/psycopg-test.py rename to tests/psycopg-test.broken.py diff --git a/wasmer-requirements.txt b/wasmer-requirements.txt index 16c756f..e82cc35 100644 --- a/wasmer-requirements.txt +++ b/wasmer-requirements.txt @@ -7,6 +7,8 @@ annotated-types==0.7.0 # via # wasmer-build-scripts (pyproject.toml) # pydantic +anyio==4.10.0 + # via starlette brotlicffi==1.1.0.0 # via wasmer-build-scripts (pyproject.toml) certifi==2025.8.3 @@ -23,22 +25,27 @@ charset-normalizer==3.4.3 # via # wasmer-build-scripts (pyproject.toml) # requests +click==8.2.1 + # via uvicorn colorama==0.4.6 ; sys_platform == 'win32' # via - # pytest + # click # qrcode cryptography==45.0.4 # via # wasmer-build-scripts (pyproject.toml) # pyopenssl +fastapi==0.116.2 + # via wasmer-build-scripts (pyproject.toml) google-crc32c==1.7.1 # via wasmer-build-scripts (pyproject.toml) +h11==0.16.0 + # via uvicorn idna==3.10 # via # wasmer-build-scripts (pyproject.toml) + # anyio # requests -iniconfig==2.1.0 - # via pytest jiter==0.10.0 # via wasmer-build-scripts (pyproject.toml) lxml==6.0.0 @@ -56,14 +63,10 @@ numpy==2.3.2 # shapely orjson==3.11.0 # via wasmer-build-scripts (pyproject.toml) -packaging==25.0 - # via pytest pandas==2.3.2 # via wasmer-build-scripts (pyproject.toml) pillow==11.3.0 # via wasmer-build-scripts (pyproject.toml) -pluggy==1.6.0 - # via pytest protobuf==6.31.1 # via wasmer-build-scripts (pyproject.toml) psycopg==3.2.9 @@ -81,11 +84,11 @@ pycryptodome==3.23.0 pycryptodomex==3.23.0 # via wasmer-build-scripts (pyproject.toml) pydantic==2.11.7 - # via wasmer-build-scripts (pyproject.toml) + # via + # wasmer-build-scripts (pyproject.toml) + # fastapi pydantic-core==2.33.2 # via pydantic -pygments==2.19.2 - # via pytest pynacl==1.5.0 # via wasmer-build-scripts (pyproject.toml) pyopenssl==25.1.0 @@ -94,8 +97,6 @@ pypandoc==1.15 # via wasmer-build-scripts (pyproject.toml) pypng==0.20250521.0 # via wasmer-build-scripts (pyproject.toml) -pytest==8.4.2 - # via wasmer-build-scripts (pyproject.toml) python-dateutil==2.9.0 # via # wasmer-build-scripts (pyproject.toml) @@ -126,6 +127,10 @@ six==1.17.0 # via # wasmer-build-scripts (pyproject.toml) # python-dateutil +sniffio==1.3.1 + # via anyio +starlette==0.48.0 + # via fastapi svgwrite==1.4.3 # via wasmer-build-scripts (pyproject.toml) tiktoken==0.9.0 @@ -133,6 +138,7 @@ tiktoken==0.9.0 typing-extensions==4.14.1 # via # wasmer-build-scripts (pyproject.toml) + # fastapi # psycopg-pool # pydantic # pydantic-core @@ -150,5 +156,7 @@ urllib3==2.5.0 # via # wasmer-build-scripts (pyproject.toml) # requests +uvicorn==0.35.0 + # via wasmer-build-scripts (pyproject.toml) uvloop==0.21.0 # via wasmer-build-scripts (pyproject.toml) diff --git a/wasmer.toml b/wasmer.toml index 217c744..cb6154c 100644 --- a/wasmer.toml +++ b/wasmer.toml @@ -1,6 +1,6 @@ [package] name = "wasmer/build-scripts" -version = "0.1.1" +version = "0.1.3" [dependencies] "wasmer/python-native" = "=0.1.11" @@ -15,10 +15,5 @@ module = "wasmer/python-native:python" runner = "wasi" [command.annotations.wasi] -main-args = ["/app/run-tests.py"] -env = [ - "PYTHONEXECUTABLE=/bin/python", - "PYTHONHOME=/cpython", - "PYTHONPATH=/opt/venv/lib/python3.13/site-packages", - "HOME=/app", -] +main-args = ["/app/main.py"] +env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "PYTHONPATH=/opt/venv/lib/python3.13/site-packages", "HOME=/app"] From e0107f9aed50e315977dc9247fcb1950c9ba2837 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 11:21:38 +0300 Subject: [PATCH 15/33] refactor: It now deploys and runs on edge --- pyproject.toml | 4 ++++ main.py => src/main.py | 2 +- run-tests.py => src/run-tests.py | 0 wasmer.toml | 12 +++++++----- 4 files changed, 12 insertions(+), 6 deletions(-) rename main.py => src/main.py (92%) rename run-tests.py => src/run-tests.py (100%) diff --git a/pyproject.toml b/pyproject.toml index eb09f91..8150eaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + [project] name = "wasmer-build-scripts" version = "0.0.0" diff --git a/main.py b/src/main.py similarity index 92% rename from main.py rename to src/main.py index 0a062ba..c63c21b 100644 --- a/main.py +++ b/src/main.py @@ -29,7 +29,7 @@ def write(self, data): @app.get("/") async def root(): return { - "message": "Build Scripts - Tests", + "message": "Build Scripts - Tests. Be aware that calling /check will take anywhere from 3-5 minutes to deliver a response. Look in the dashboard logs for progress.", "version": "0.1.0", "endpoints": [ "/", diff --git a/run-tests.py b/src/run-tests.py similarity index 100% rename from run-tests.py rename to src/run-tests.py diff --git a/wasmer.toml b/wasmer.toml index cb6154c..59f0839 100644 --- a/wasmer.toml +++ b/wasmer.toml @@ -1,19 +1,21 @@ [package] name = "wasmer/build-scripts" -version = "0.1.3" +version = "0.1.6" +entrypoint = "start" [dependencies] "wasmer/python-native" = "=0.1.11" [fs] -"/app" = "./" -"/cpython/lib/python3.13/site-packages" = "./wasix-site-packages" +"/app" = "./src" +"/tests" = "./tests" +"/opt/venv/packages" = "./wasix-site-packages" [[command]] -name = "script" +name = "start" module = "wasmer/python-native:python" runner = "wasi" [command.annotations.wasi] main-args = ["/app/main.py"] -env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "PYTHONPATH=/opt/venv/lib/python3.13/site-packages", "HOME=/app"] +env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "PYTHONPATH=/opt/venv/packages", "HOME=/app"] From 24467a3792c29381fdde0a555292eb084d6249d9 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 14:33:55 +0300 Subject: [PATCH 16/33] feat: Added /check/ and added run-all-tests-via-api.py Although the /check endpoint worked just fine locally, it times out the workload on edge. So we need to be able to call each test one by one. So in addition to this, I added a /list endpoint which lists all tests, then a /check/ endpoint which calls a singular test. This allows us to run just a specific test, if need be. Also added instructions on how this works to README.md --- README.md | 311 ++++++++++++++++++++------------------- run-all-tests-via-api.py | 121 +++++++++++++++ src/main.py | 106 ++++++++++++- src/run-tests.py | 7 +- wasmer.toml | 2 +- 5 files changed, 395 insertions(+), 152 deletions(-) create mode 100755 run-all-tests-via-api.py diff --git a/README.md b/README.md index 7ef9809..b79881e 100644 --- a/README.md +++ b/README.md @@ -84,10 +84,10 @@ Buildscripts to build numpy and other wheels for wasix. For convenience, this pa The build script is controlled by the following environment variables: -* `CC`, `CXX`, `AR`, `LD`, `RANLIB`, etc... : The cross-compiler tools. These should all be normal clang tools, but target wasm32-wasix by default and use the wasix sysroot. -* `WASIX_SYSROOT`: The path to the wasix sysroot that is used by the toolchain. Libraries will get installed here when you run `make install` or when they are required to build a package. -* `INSTALL_DIR`: The path to the python library path. Wheels will get installed here when you run `make install`. -* `WASMER`: The path to the wasmer binary. You must have it registered to handle wasm files as binfmt_misc. You can do this with `sudo $WASMER binfmt reregister`. +- `CC`, `CXX`, `AR`, `LD`, `RANLIB`, etc... : The cross-compiler tools. These should all be normal clang tools, but target wasm32-wasix by default and use the wasix sysroot. +- `WASIX_SYSROOT`: The path to the wasix sysroot that is used by the toolchain. Libraries will get installed here when you run `make install` or when they are required to build a package. +- `INSTALL_DIR`: The path to the python library path. Wheels will get installed here when you run `make install`. +- `WASMER`: The path to the wasmer binary. You must have it registered to handle wasm files as binfmt_misc. You can do this with `sudo $WASMER binfmt reregister`. The easiest way to setup all the environment variables is to activate the wasix-clang environment using `source wasix-clang/activate`. @@ -154,111 +154,111 @@ Here is a list of the versions of the wheels and libraries that are included in #### Wheels -* numpy: numpy/numpy main -* markupsafe: 3.0.2 -* pandas: 2.3.2 -* pytz: 2025.2 -* dateutil: 2.9.0 -* tzdata: 2025.2 -* six: 1.17.0 -* msgpack: 1.1.0 -* pycryptodome: 3.23.0 -* pycryptodomex: 3.23.0 -* pyzbar: 0.1.9 -* cpython: 3.1.2 -* pypandoc: 1.15 -* pypandoc_binary: 1.15 -* psycopg: 3.2.9 -* psycopg-binary: 3.2.9 -* psycopg-pool: pool-3.2.6 -* brotlicffi: 1.1.0.0 -* cffi: 1.17.1 -* pillow: 11.3.0 -* matplotlib: 3.10.6 -* uvloop: 0.21.0 -* mysqlclient: 2.2.7 -* python-qrcode: 8.2 -* pycparser: 2.22 -* pydantic: 2.11.7 -* typing_extensions: 4.14.1 -* typing-inspection: 0.4.1 -* annotated-types: 0.7.0 -* shapely: 2.1.1 -* mrab-regex: 2025.7.31 -* lxml: 6.0.0 -* protobuf: 31.1 -* grpc: 1.74.1 -* numpy: 1.26.5 -* numpy: 2.0.2 -* numpy: 2.3.2 -* python-crc32c: 1.7.1 -* requests: 2.32.4 -* urllib3: 2.5.0 -* idna: 3.10 -* certifi: 2025.08.03 -* charset-normalizer: 3.4.3 -* pypng: 0.20250521.0 -* pyarrow: 19.0.1 -* pyarrow: 21.0.0 -* packaging: 25.0 -* pyparsing: 3.2.3 -* cycler: 0.12.1 -* kiwisolver: 1.4.9 -* contourpy: 1.3.3 -* pyopenssl: 25.1.0 -* aspw: 3.50.4.0 +- numpy: numpy/numpy main +- markupsafe: 3.0.2 +- pandas: 2.3.2 +- pytz: 2025.2 +- dateutil: 2.9.0 +- tzdata: 2025.2 +- six: 1.17.0 +- msgpack: 1.1.0 +- pycryptodome: 3.23.0 +- pycryptodomex: 3.23.0 +- pyzbar: 0.1.9 +- cpython: 3.1.2 +- pypandoc: 1.15 +- pypandoc_binary: 1.15 +- psycopg: 3.2.9 +- psycopg-binary: 3.2.9 +- psycopg-pool: pool-3.2.6 +- brotlicffi: 1.1.0.0 +- cffi: 1.17.1 +- pillow: 11.3.0 +- matplotlib: 3.10.6 +- uvloop: 0.21.0 +- mysqlclient: 2.2.7 +- python-qrcode: 8.2 +- pycparser: 2.22 +- pydantic: 2.11.7 +- typing_extensions: 4.14.1 +- typing-inspection: 0.4.1 +- annotated-types: 0.7.0 +- shapely: 2.1.1 +- mrab-regex: 2025.7.31 +- lxml: 6.0.0 +- protobuf: 31.1 +- grpc: 1.74.1 +- numpy: 1.26.5 +- numpy: 2.0.2 +- numpy: 2.3.2 +- python-crc32c: 1.7.1 +- requests: 2.32.4 +- urllib3: 2.5.0 +- idna: 3.10 +- certifi: 2025.08.03 +- charset-normalizer: 3.4.3 +- pypng: 0.20250521.0 +- pyarrow: 19.0.1 +- pyarrow: 21.0.0 +- packaging: 25.0 +- pyparsing: 3.2.3 +- cycler: 0.12.1 +- kiwisolver: 1.4.9 +- contourpy: 1.3.3 +- pyopenssl: 25.1.0 +- aspw: 3.50.4.0 psycopg3-c is just the sdist of psycopg3-binary #### Libraries -* libzbar: 0.23.93 -* libffi: wasix-org/libffi main -* pandoc: haskell-wasm/pandoc wasm -* postgresql: 17.5 -* brotli: 1.1.0 -* zlib: develop - * 1.3.1 does not have proper cmake support, so we are using develop for now -* libjpeg-turbo: 3.1.1 -* xz: 5.8.1 -* libtiff: 4.7.0 -* libwebp: 1.5.0 -* giflib: 5.2.2 -* libpng: 1.6.50 -* SDL: 3.2.16 - * SDL has all subsystems disabled -* openjpeg: 2.5.3 -* libuv: 1.51.0 -* mariadb-connector-c: 3.4.6 -* openssl: 3.5.1 -* bzip2: 1.0.8 -* util-linux: 2.41.1 - * We only build libuuid from util-linux -* openssh: 10.0p2 -* dropbear: 2025.88 -* tinyxml2: 11.0.0 -* geos: 3.13.1 -* libxslt: 1.1.43 -* libxml2: 2.14.5 -* google-crc32c: 1.1.2 -* arrow: 19.0.1 -* arrow: 21.0.0 -* rapidjson: 1.1.0 -* icu: 77.1 -* readline: 8.2 -* ncurses: 6.4.20230225 -* curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 - * This was the current commit in the wasix fork. We can probably update to the next release -* pycurl: 7.45.6 -* sqlite: 3.50.4 -* wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 - * The commit before we removed fork. A few packages don't build if fork is not defined. -* libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - * This is the commit that is currently used by wasix-libc -* compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - * This is the commit that is currently used by wasix-libc - +- libzbar: 0.23.93 +- libffi: wasix-org/libffi main +- pandoc: haskell-wasm/pandoc wasm +- postgresql: 17.5 +- brotli: 1.1.0 +- zlib: develop + - 1.3.1 does not have proper cmake support, so we are using develop for now +- libjpeg-turbo: 3.1.1 +- xz: 5.8.1 +- libtiff: 4.7.0 +- libwebp: 1.5.0 +- giflib: 5.2.2 +- libpng: 1.6.50 +- SDL: 3.2.16 + - SDL has all subsystems disabled +- openjpeg: 2.5.3 +- libuv: 1.51.0 +- mariadb-connector-c: 3.4.6 +- openssl: 3.5.1 +- bzip2: 1.0.8 +- util-linux: 2.41.1 + - We only build libuuid from util-linux +- openssh: 10.0p2 +- dropbear: 2025.88 +- tinyxml2: 11.0.0 +- geos: 3.13.1 +- libxslt: 1.1.43 +- libxml2: 2.14.5 +- google-crc32c: 1.1.2 +- arrow: 19.0.1 +- arrow: 21.0.0 +- rapidjson: 1.1.0 +- icu: 77.1 +- readline: 8.2 +- ncurses: 6.4.20230225 +- curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 + - This was the current commit in the wasix fork. We can probably update to the next release +- pycurl: 7.45.6 +- sqlite: 3.50.4 +- wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 + - The commit before we removed fork. A few packages don't build if fork is not defined. +- libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + - This is the commit that is currently used by wasix-libc +- compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + - This is the commit that is currently used by wasix-libc + ## Tests @@ -269,35 +269,48 @@ So there are two ways of running the tests, natively and with wasmer: Requirements: -* Python 3.10+ -* `python -m venv env` -* `source venv/bin/activate` -* `pip install .` -* `pip install pytest` +- Python 3.10+ +- `python -m venv env` +- `source venv/bin/activate` +- `pip install .` +- `pip install pytest` I'm sure it's possible to do via uv, or poetry, or many other ways. But this works also. Run tests: -* `python -m pytest` -* Discovers files matching `*-test.py`, `*_test.py`, and `test_*.py` under `tests/`. -* Files ending with `.skip.py` are ignored. -* Files ending with `-broken.py` are marked as expected failures (strict). If they pass, the run reports XPASS and fails. +- `python -m pytest` +- Discovers files matching `*-test.py`, `*_test.py`, and `test_*.py` under `tests/`. +- Files ending with `.skip.py` are ignored. +- Files ending with `-broken.py` are marked as expected failures (strict). If they pass, the run reports XPASS and fails. ### Wasmer These are the instructions as of 2025-10, but there's a shipit looming, which may optimize the flow. -If this doesn't work though, we have a bug: Requirements: -* wasmer 6.1.0-rc.3+ +- wasmer 6.1.0-rc.3+ Run tests: -* `uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt` -* `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile` -* `wasmer run .` +- `uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt` +- `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile` +- `TEST_DIR=/tests/ wasmer run . --registry=wasmer.wtf --net --forward-host-env` +- `curl localhost:8081/check` + +This will run all tests via fastapi. + +You may also run each test individually by: + +- `curl localhost:80801/list` +- `curl localhost:8081/check/` + +This is needed when testing on edge, since `.../check` times out the workload. +In conjunction with this, there is a convenience script which runs all tests each in a separate query. + +So you may also run `./run-all-tests-via-api.py --host --port `. +This is intended to be run to validate package functionaltiy on edge, as each test becomes a separate workload. ### Notes @@ -325,7 +338,7 @@ to check which python libraries depend on shared libs. We try to keep that to a ### Structure - psycopg3-c is just the sdist of psycopg3-binary #### Libraries -- libzbar: 0.23.93 -- libffi: wasix-org/libffi main -- pandoc: haskell-wasm/pandoc wasm -- postgresql: 17.5 -- brotli: 1.1.0 -- zlib: develop - - 1.3.1 does not have proper cmake support, so we are using develop for now -- libjpeg-turbo: 3.1.1 -- xz: 5.8.1 -- libtiff: 4.7.0 -- libwebp: 1.5.0 -- giflib: 5.2.2 -- libpng: 1.6.50 -- SDL: 3.2.16 - - SDL has all subsystems disabled -- openjpeg: 2.5.3 -- libuv: 1.51.0 -- mariadb-connector-c: 3.4.6 -- openssl: 3.5.1 -- bzip2: 1.0.8 -- util-linux: 2.41.1 - - We only build libuuid from util-linux -- openssh: 10.0p2 -- dropbear: 2025.88 -- tinyxml2: 11.0.0 -- geos: 3.13.1 -- libxslt: 1.1.43 -- libxml2: 2.14.5 -- google-crc32c: 1.1.2 -- arrow: 19.0.1 -- arrow: 21.0.0 -- rapidjson: 1.1.0 -- icu: 77.1 -- readline: 8.2 -- ncurses: 6.4.20230225 -- curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 - - This was the current commit in the wasix fork. We can probably update to the next release -- pycurl: 7.45.6 -- sqlite: 3.50.4 -- wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 - - The commit before we removed fork. A few packages don't build if fork is not defined. -- libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - - This is the commit that is currently used by wasix-libc -- compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d - - This is the commit that is currently used by wasix-libc - +* libzbar: 0.23.93 +* libffi: wasix-org/libffi main +* pandoc: haskell-wasm/pandoc wasm +* postgresql: 17.5 +* brotli: 1.1.0 +* zlib: develop + * 1.3.1 does not have proper cmake support, so we are using develop for now +* libjpeg-turbo: 3.1.1 +* xz: 5.8.1 +* libtiff: 4.7.0 +* libwebp: 1.5.0 +* giflib: 5.2.2 +* libpng: 1.6.50 +* SDL: 3.2.16 + * SDL has all subsystems disabled +* openjpeg: 2.5.3 +* libuv: 1.51.0 +* mariadb-connector-c: 3.4.6 +* openssl: 3.5.1 +* bzip2: 1.0.8 +* util-linux: 2.41.1 + * We only build libuuid from util-linux +* openssh: 10.0p2 +* dropbear: 2025.88 +* tinyxml2: 11.0.0 +* geos: 3.13.1 +* libxslt: 1.1.43 +* libxml2: 2.14.5 +* google-crc32c: 1.1.2 +* arrow: 19.0.1 +* arrow: 21.0.0 +* rapidjson: 1.1.0 +* icu: 77.1 +* readline: 8.2 +* ncurses: 6.4.20230225 +* curl: curl/curl ab18c04218ff316cd67b1e928c5cee579b2f66a0 + * This was the current commit in the wasix fork. We can probably update to the next release +* pycurl: 7.45.6 +* sqlite: 3.50.4 +* wasix-libc: wasix-org/wasix-libc e4e5cfcf1009e690207821e9b3a63cdce2ee9e86 + * The commit before we removed fork. A few packages don't build if fork is not defined. +* libcxx: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + * This is the commit that is currently used by wasix-libc +* compiler-rt: wasix-org/llvm-project 6bb93a243f6d15855f485f5aec3810d9e2de150d + * This is the commit that is currently used by wasix-libc + ## Tests From 424cfb61e637077b5d7d66e0b5bd974c53c5a05b Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 17:16:40 +0300 Subject: [PATCH 20/33] Apply suggestion from @zebreus Co-authored-by: Zebreus --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 076b621..c780a22 100644 --- a/README.md +++ b/README.md @@ -269,7 +269,7 @@ So there are two ways of running the tests, natively and with wasmer: Requirements: -- Python 3.10+ +- Python 3.13 - `python -m venv env` - `source venv/bin/activate` - `pip install .` From 62348cd307546c505fd1008df3a8d85ff41c5ae9 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 17:25:20 +0300 Subject: [PATCH 21/33] cleanup: Moved new testing system into directory testing --- .gitignore | 1 + README.md | 2 +- pyproject.toml => testing/pyproject.toml | 0 .../run-all-tests-via-api.py | 0 {src => testing/src}/main.py | 0 {src => testing/src}/run-tests.py | 0 wasmer.toml => testing/wasmer.toml | 9 +- wasmer-requirements.txt | 162 ------------------ 8 files changed, 9 insertions(+), 165 deletions(-) rename pyproject.toml => testing/pyproject.toml (100%) rename run-all-tests-via-api.py => testing/run-all-tests-via-api.py (100%) rename {src => testing/src}/main.py (100%) rename {src => testing/src}/run-tests.py (100%) rename wasmer.toml => testing/wasmer.toml (70%) delete mode 100644 wasmer-requirements.txt diff --git a/.gitignore b/.gitignore index 07815c0..e58a994 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ __pycache__ /pkgs/*.prepared /pkgs/*.sysroot wasix-site-packages +wasmer-requirements.txt diff --git a/README.md b/README.md index c780a22..44a4501 100644 --- a/README.md +++ b/README.md @@ -296,7 +296,7 @@ Run tests: - `uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt` - `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile` -- `TEST_DIR=/tests/ wasmer run . --registry=wasmer.wtf --net --forward-host-env` +- `TEST_DIR=../tests/ wasmer run . --registry=wasmer.wtf --net --forward-host-env` - `curl localhost:8081/check` This will run all tests via fastapi. diff --git a/pyproject.toml b/testing/pyproject.toml similarity index 100% rename from pyproject.toml rename to testing/pyproject.toml diff --git a/run-all-tests-via-api.py b/testing/run-all-tests-via-api.py similarity index 100% rename from run-all-tests-via-api.py rename to testing/run-all-tests-via-api.py diff --git a/src/main.py b/testing/src/main.py similarity index 100% rename from src/main.py rename to testing/src/main.py diff --git a/src/run-tests.py b/testing/src/run-tests.py similarity index 100% rename from src/run-tests.py rename to testing/src/run-tests.py diff --git a/wasmer.toml b/testing/wasmer.toml similarity index 70% rename from wasmer.toml rename to testing/wasmer.toml index 7d13586..17203be 100644 --- a/wasmer.toml +++ b/testing/wasmer.toml @@ -8,7 +8,7 @@ entrypoint = "start" [fs] "/app" = "./src" -"/tests" = "./tests" +"/tests" = "../tests" "/opt/venv/packages" = "./wasix-site-packages" [[command]] @@ -18,4 +18,9 @@ runner = "wasi" [command.annotations.wasi] main-args = ["/app/main.py"] -env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "PYTHONPATH=/opt/venv/packages", "HOME=/app"] +env = [ + "PYTHONEXECUTABLE=/bin/python", + "PYTHONHOME=/cpython", + "PYTHONPATH=/opt/venv/packages", + "HOME=/app", +] diff --git a/wasmer-requirements.txt b/wasmer-requirements.txt deleted file mode 100644 index e82cc35..0000000 --- a/wasmer-requirements.txt +++ /dev/null @@ -1,162 +0,0 @@ -# This file was autogenerated by uv via the following command: -# uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt ---index-url https://pypi.org/simple ---extra-index-url https://pythonindex.wasix.org/simple - -annotated-types==0.7.0 - # via - # wasmer-build-scripts (pyproject.toml) - # pydantic -anyio==4.10.0 - # via starlette -brotlicffi==1.1.0.0 - # via wasmer-build-scripts (pyproject.toml) -certifi==2025.8.3 - # via - # wasmer-build-scripts (pyproject.toml) - # requests -cffi==1.17.1 - # via - # wasmer-build-scripts (pyproject.toml) - # brotlicffi - # cryptography - # pynacl -charset-normalizer==3.4.3 - # via - # wasmer-build-scripts (pyproject.toml) - # requests -click==8.2.1 - # via uvicorn -colorama==0.4.6 ; sys_platform == 'win32' - # via - # click - # qrcode -cryptography==45.0.4 - # via - # wasmer-build-scripts (pyproject.toml) - # pyopenssl -fastapi==0.116.2 - # via wasmer-build-scripts (pyproject.toml) -google-crc32c==1.7.1 - # via wasmer-build-scripts (pyproject.toml) -h11==0.16.0 - # via uvicorn -idna==3.10 - # via - # wasmer-build-scripts (pyproject.toml) - # anyio - # requests -jiter==0.10.0 - # via wasmer-build-scripts (pyproject.toml) -lxml==6.0.0 - # via wasmer-build-scripts (pyproject.toml) -markupsafe==3.0.2 - # via wasmer-build-scripts (pyproject.toml) -msgpack==1.1.1rc1 - # via wasmer-build-scripts (pyproject.toml) -mysqlclient==2.2.7 - # via wasmer-build-scripts (pyproject.toml) -numpy==2.3.2 - # via - # wasmer-build-scripts (pyproject.toml) - # pandas - # shapely -orjson==3.11.0 - # via wasmer-build-scripts (pyproject.toml) -pandas==2.3.2 - # via wasmer-build-scripts (pyproject.toml) -pillow==11.3.0 - # via wasmer-build-scripts (pyproject.toml) -protobuf==6.31.1 - # via wasmer-build-scripts (pyproject.toml) -psycopg==3.2.9 - # via wasmer-build-scripts (pyproject.toml) -psycopg-binary==3.2.9 ; implementation_name != 'pypy' - # via psycopg -psycopg-pool==3.2.6 - # via wasmer-build-scripts (pyproject.toml) -pycparser==2.22 - # via - # wasmer-build-scripts (pyproject.toml) - # cffi -pycryptodome==3.23.0 - # via wasmer-build-scripts (pyproject.toml) -pycryptodomex==3.23.0 - # via wasmer-build-scripts (pyproject.toml) -pydantic==2.11.7 - # via - # wasmer-build-scripts (pyproject.toml) - # fastapi -pydantic-core==2.33.2 - # via pydantic -pynacl==1.5.0 - # via wasmer-build-scripts (pyproject.toml) -pyopenssl==25.1.0 - # via wasmer-build-scripts (pyproject.toml) -pypandoc==1.15 - # via wasmer-build-scripts (pyproject.toml) -pypng==0.20250521.0 - # via wasmer-build-scripts (pyproject.toml) -python-dateutil==2.9.0 - # via - # wasmer-build-scripts (pyproject.toml) - # pandas -pytz==2025.2 - # via - # wasmer-build-scripts (pyproject.toml) - # pandas -pyyaml==6.0.2 - # via wasmer-build-scripts (pyproject.toml) -pyzbar==0.1.9 - # via wasmer-build-scripts (pyproject.toml) -qrcode==8.2 - # via wasmer-build-scripts (pyproject.toml) -regex==2025.7.31 - # via - # wasmer-build-scripts (pyproject.toml) - # tiktoken -requests==2.32.4 - # via - # wasmer-build-scripts (pyproject.toml) - # tiktoken -rpds-py==0.26.0 - # via wasmer-build-scripts (pyproject.toml) -shapely==2.1.1 - # via wasmer-build-scripts (pyproject.toml) -six==1.17.0 - # via - # wasmer-build-scripts (pyproject.toml) - # python-dateutil -sniffio==1.3.1 - # via anyio -starlette==0.48.0 - # via fastapi -svgwrite==1.4.3 - # via wasmer-build-scripts (pyproject.toml) -tiktoken==0.9.0 - # via wasmer-build-scripts (pyproject.toml) -typing-extensions==4.14.1 - # via - # wasmer-build-scripts (pyproject.toml) - # fastapi - # psycopg-pool - # pydantic - # pydantic-core - # typing-inspection -typing-inspection==0.4.1 - # via - # wasmer-build-scripts (pyproject.toml) - # pydantic -tzdata==2025.2 - # via - # wasmer-build-scripts (pyproject.toml) - # pandas - # psycopg -urllib3==2.5.0 - # via - # wasmer-build-scripts (pyproject.toml) - # requests -uvicorn==0.35.0 - # via wasmer-build-scripts (pyproject.toml) -uvloop==0.21.0 - # via wasmer-build-scripts (pyproject.toml) From 250060cd79c5bfa4bf133fdc72d8f6913b3c5833 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 17:27:27 +0300 Subject: [PATCH 22/33] cleanup: Removed try-catch on pysql and mysql tests to fish for linker errors --- tests/mysqlclient-test.py | 14 +++++--------- tests/psycopg-pool-test.broken.py | 28 ++++++++++++---------------- tests/psycopg-test.broken.py | 26 +++++++++++--------------- 3 files changed, 28 insertions(+), 40 deletions(-) diff --git a/tests/mysqlclient-test.py b/tests/mysqlclient-test.py index c5de075..b3c4546 100644 --- a/tests/mysqlclient-test.py +++ b/tests/mysqlclient-test.py @@ -2,15 +2,11 @@ from MySQLdb import _mysql -try: - db = _mysql.connect(host="127.0.0.1", port=3306, user="root", password="password") +db = _mysql.connect(host="127.0.0.1", port=3306, user="root", password="password") - db.query("SELECT VERSION()") +db.query("SELECT VERSION()") - result = db.use_result() - row = result.fetch_row() +result = db.use_result() +row = result.fetch_row() - print("MySQL Server Version:", row[0][0]) -except: - # It runs code, good enough until we can run a mysqlserver in wasix! - pass +print("MySQL Server Version:", row[0][0]) diff --git a/tests/psycopg-pool-test.broken.py b/tests/psycopg-pool-test.broken.py index 6f91fde..af7c887 100644 --- a/tests/psycopg-pool-test.broken.py +++ b/tests/psycopg-pool-test.broken.py @@ -4,19 +4,15 @@ from psycopg_pool import ConnectionPool -try: - pool = ConnectionPool( - dbname=os.environ.get("POSTGRES_DB", "docker-local"), - user=os.environ.get("POSTGRES_USER", "postgres"), - password=os.environ.get("POSTGRES_PASSWORD", "securesecret"), - host=os.environ.get("DB_HOST", "0.0.0.0"), - port=os.environ.get("DB_PORT", "5432"), - ) - with pool.connection() as conn: - with conn.cursor() as cur: - cur.execute("SELECT version();") - version = cur.fetchone() - print("PostgreSQL version:", version[0]) -except: - # It runs code, good enough until we can run a psql in wasix! - pass +pool = ConnectionPool( + dbname=os.environ.get("POSTGRES_DB", "docker-local"), + user=os.environ.get("POSTGRES_USER", "postgres"), + password=os.environ.get("POSTGRES_PASSWORD", "securesecret"), + host=os.environ.get("DB_HOST", "0.0.0.0"), + port=os.environ.get("DB_PORT", "5432"), +) +with pool.connection() as conn: + with conn.cursor() as cur: + cur.execute("SELECT version();") + version = cur.fetchone() + print("PostgreSQL version:", version[0]) diff --git a/tests/psycopg-test.broken.py b/tests/psycopg-test.broken.py index cc1fb72..df33920 100644 --- a/tests/psycopg-test.broken.py +++ b/tests/psycopg-test.broken.py @@ -4,18 +4,14 @@ import psycopg -try: - conn = psycopg.connect( - dbname=os.environ.get("POSTGRES_DB", "docker-local"), - user=os.environ.get("POSTGRES_USER", "postgres"), - password=os.environ.get("POSTGRES_PASSWORD", "securesecret"), - host=os.environ.get("DB_HOST", "0.0.0.0"), - port=os.environ.get("DB_PORT", "5432"), - ) - cur = conn.cursor() - cur.execute("SELECT version();") - version = cur.fetchone() - print("PostgreSQL version:", version[0]) -except: - # It runs code, good enough until we can run a psql in wasix! - pass +conn = psycopg.connect( + dbname=os.environ.get("POSTGRES_DB", "docker-local"), + user=os.environ.get("POSTGRES_USER", "postgres"), + password=os.environ.get("POSTGRES_PASSWORD", "securesecret"), + host=os.environ.get("DB_HOST", "0.0.0.0"), + port=os.environ.get("DB_PORT", "5432"), +) +cur = conn.cursor() +cur.execute("SELECT version();") +version = cur.fetchone() +print("PostgreSQL version:", version[0]) From a83b06fb2bd00745750bd9e7669e9d9920410183 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 17:41:53 +0300 Subject: [PATCH 23/33] tweaks: Test pipeline should in theory now work --- .github/workflows/test-python-index.yaml | 3 ++- testing/wasmer.toml | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-python-index.yaml b/.github/workflows/test-python-index.yaml index 60695b3..5a1c479 100644 --- a/.github/workflows/test-python-index.yaml +++ b/.github/workflows/test-python-index.yaml @@ -42,6 +42,7 @@ jobs: run: curl https://get.wasmer.io -sSfL | sh -s ${{ inputs.wasmer-version }} - name: Setup project for wasmer run: | + cd testing uv pip compile pyproject.toml \ --python-version=${{ inputs.python-version }} \ --universal \ @@ -59,4 +60,4 @@ jobs: --compile # Ensure tests pass on wasmer - name: Run tests (wasmer) - run: wasmer run . + run: wasmer run . --registry=wasmer.wtf --command-name=test --net diff --git a/testing/wasmer.toml b/testing/wasmer.toml index 17203be..f4e5b25 100644 --- a/testing/wasmer.toml +++ b/testing/wasmer.toml @@ -1,7 +1,7 @@ [package] name = "wasmer/build-scripts" version = "0.1.10" -entrypoint = "start" +entrypoint = "server" [dependencies] "wasmer/python-native" = "=0.1.11" @@ -12,7 +12,7 @@ entrypoint = "start" "/opt/venv/packages" = "./wasix-site-packages" [[command]] -name = "start" +name = "server" module = "wasmer/python-native:python" runner = "wasi" @@ -24,3 +24,17 @@ env = [ "PYTHONPATH=/opt/venv/packages", "HOME=/app", ] + +[[command]] +name = "test" +module = "wasmer/python-native:python" +runner = "wasi" + +[command.annotations.wasi] +main-args = ["/app/run-tests.py"] +env = [ + "PYTHONEXECUTABLE=/bin/python", + "PYTHONHOME=/cpython", + "PYTHONPATH=/opt/venv/packages", + "HOME=/app", +] From 7a4e785ff0f649cf93eb04c4dd80238ce2549cbc Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 17:48:23 +0300 Subject: [PATCH 24/33] tweaks: ... theory was incorrect, now it should work --- .github/workflows/test-python-index.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-python-index.yaml b/.github/workflows/test-python-index.yaml index 5a1c479..80ffce9 100644 --- a/.github/workflows/test-python-index.yaml +++ b/.github/workflows/test-python-index.yaml @@ -28,13 +28,14 @@ jobs: python-version: ${{ inputs.python-version }} - name: Setup environment run: | + cd testing sudo apt-get install libzbar0 pip install . - pip install pytest # Validate that the python tests themselves are valid - - name: Run tests - run: pytest - + - name: Run tests natively + # This should hopefully be temporary + continue-on-error: true + run: TEST_DIR=../ python src/run_test.py # Secondly, validate that the tests pass within wasmer as well - name: Install uv run: curl -LsSf https://astral.sh/uv/install.sh | sh @@ -42,7 +43,6 @@ jobs: run: curl https://get.wasmer.io -sSfL | sh -s ${{ inputs.wasmer-version }} - name: Setup project for wasmer run: | - cd testing uv pip compile pyproject.toml \ --python-version=${{ inputs.python-version }} \ --universal \ From d805739a711056a649c6cf65422aa40475fcc3b6 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 17:52:43 +0300 Subject: [PATCH 25/33] fix: Paths should now be OK --- .github/workflows/test-python-index.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-python-index.yaml b/.github/workflows/test-python-index.yaml index 80ffce9..43b1712 100644 --- a/.github/workflows/test-python-index.yaml +++ b/.github/workflows/test-python-index.yaml @@ -35,7 +35,7 @@ jobs: - name: Run tests natively # This should hopefully be temporary continue-on-error: true - run: TEST_DIR=../ python src/run_test.py + run: TEST_DIR=../tests/ python src/run_test.py # Secondly, validate that the tests pass within wasmer as well - name: Install uv run: curl -LsSf https://astral.sh/uv/install.sh | sh @@ -43,6 +43,7 @@ jobs: run: curl https://get.wasmer.io -sSfL | sh -s ${{ inputs.wasmer-version }} - name: Setup project for wasmer run: | + cd testing uv pip compile pyproject.toml \ --python-version=${{ inputs.python-version }} \ --universal \ @@ -60,4 +61,4 @@ jobs: --compile # Ensure tests pass on wasmer - name: Run tests (wasmer) - run: wasmer run . --registry=wasmer.wtf --command-name=test --net + run: cd testing && wasmer run . --registry=wasmer.wtf --command-name=test --net From e4dab79d8bad0a9d1e42f5a7f3e562a66f0eb41a Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 18:46:30 +0300 Subject: [PATCH 26/33] fix: Path to wasmer --- .github/workflows/test-python-index.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-python-index.yaml b/.github/workflows/test-python-index.yaml index 43b1712..1d07010 100644 --- a/.github/workflows/test-python-index.yaml +++ b/.github/workflows/test-python-index.yaml @@ -61,4 +61,4 @@ jobs: --compile # Ensure tests pass on wasmer - name: Run tests (wasmer) - run: cd testing && wasmer run . --registry=wasmer.wtf --command-name=test --net + run: cd testing && /home/runner/.wasmer/bin/wasmer run . --registry=wasmer.wtf --command-name=test --net From 953ffe4ac79e4bd9d6dcf2477abcb8c074f3d8eb Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 19:13:58 +0300 Subject: [PATCH 27/33] tweak: Fix path for running python tests natively --- .github/workflows/test-python-index.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-python-index.yaml b/.github/workflows/test-python-index.yaml index 1d07010..16bcf49 100644 --- a/.github/workflows/test-python-index.yaml +++ b/.github/workflows/test-python-index.yaml @@ -33,9 +33,9 @@ jobs: pip install . # Validate that the python tests themselves are valid - name: Run tests natively - # This should hopefully be temporary - continue-on-error: true - run: TEST_DIR=../tests/ python src/run_test.py + run: | + cd testing + TEST_DIR=../tests/ python src/run-tests.py # Secondly, validate that the tests pass within wasmer as well - name: Install uv run: curl -LsSf https://astral.sh/uv/install.sh | sh From 1ed415e6edcb893c0a6d076ad1d1c05d978a4dac Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 19:23:59 +0300 Subject: [PATCH 28/33] feat: Re-enable contine-on-error for native tests --- .github/workflows/test-python-index.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-python-index.yaml b/.github/workflows/test-python-index.yaml index 16bcf49..a504fe0 100644 --- a/.github/workflows/test-python-index.yaml +++ b/.github/workflows/test-python-index.yaml @@ -33,6 +33,7 @@ jobs: pip install . # Validate that the python tests themselves are valid - name: Run tests natively + continue-on-error: true run: | cd testing TEST_DIR=../tests/ python src/run-tests.py From fbcfe1941ec8c262ac70d03e280ebc862aa88319 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 19:24:19 +0300 Subject: [PATCH 29/33] revert: Pipeline now runs only as last step of normal release --- .github/workflows/generate-index.yml | 113 +++++++++++++-------------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/.github/workflows/generate-index.yml b/.github/workflows/generate-index.yml index 6afb9fa..5d5c254 100644 --- a/.github/workflows/generate-index.yml +++ b/.github/workflows/generate-index.yml @@ -4,11 +4,10 @@ on: push: branches: - main # Re-generate index on every change to the default branch - - qa-209-convert-build-scripts-into-a-python-test-project - # paths: - # - artifacts/*.whl - # - generate-index.py - # - python-wasix-binaries + paths: + - artifacts/*.whl + - generate-index.py + - python-wasix-binaries workflow_dispatch: # Minimal permissions required by the official Pages deployment action @@ -23,58 +22,58 @@ concurrency: cancel-in-progress: true jobs: - # build: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout source - # uses: actions/checkout@v4 - # with: - # lfs: true - # submodules: false - # sparse-checkout: | - # # Only checkout the files needed to generate the index - # artifacts - # generate-index.py - # python-wasix-binaries - # - name: Checkout python-wasix-binaries - # run: | - # git submodule update --init python-wasix-binaries - # # Install a fast Python environment + the `uv` CLI (with cache) - # - name: Setup uv & Python - # uses: astral-sh/setup-uv@v6 - # with: - # # Install the runner's system Python (faster than compiling) - # python-version: "3.12" - # enable-cache: true - # # Reuse cache when lockfile or project metadata didn’t change - # cache-dependency-glob: | - # generate-index.py - # - name: Generate static package index - # run: | - # ./generate-index.py - # - # # Upload the "dist" folder produced by the script so it can be - # # deployed by the next job. - # - name: Upload Pages artifact - # uses: actions/upload-pages-artifact@v3 - # with: - # path: dist - # - # deploy: - # needs: build - # runs-on: ubuntu-latest - # environment: - # name: github-pages # default Pages environment - # url: ${{ steps.deploy.outputs.page_url }} - # # Job-level permissions must explicitly include the same set - # permissions: - # contents: read - # pages: write - # id-token: write - # steps: - # - name: Deploy to GitHub Pages - # id: deploy - # uses: actions/deploy-pages@v4 + build: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + lfs: true + submodules: false + sparse-checkout: | + # Only checkout the files needed to generate the index + artifacts + generate-index.py + python-wasix-binaries + - name: Checkout python-wasix-binaries + run: | + git submodule update --init python-wasix-binaries + # Install a fast Python environment + the `uv` CLI (with cache) + - name: Setup uv & Python + uses: astral-sh/setup-uv@v6 + with: + # Install the runner's system Python (faster than compiling) + python-version: "3.12" + enable-cache: true + # Reuse cache when lockfile or project metadata didn’t change + cache-dependency-glob: | + generate-index.py + - name: Generate static package index + run: | + ./generate-index.py + + # Upload the "dist" folder produced by the script so it can be + # deployed by the next job. + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: dist + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages # default Pages environment + url: ${{ steps.deploy.outputs.page_url }} + # Job-level permissions must explicitly include the same set + permissions: + contents: read + pages: write + id-token: write + steps: + - name: Deploy to GitHub Pages + id: deploy + uses: actions/deploy-pages@v4 test: strategy: From b3ea1a97a2e5ef30def0698aa7e4d98b421603aa Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 19:26:27 +0300 Subject: [PATCH 30/33] tweak: Reset formatting to not joink git blame on merge commit --- .github/workflows/generate-index.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/generate-index.yml b/.github/workflows/generate-index.yml index 5d5c254..05cf00c 100644 --- a/.github/workflows/generate-index.yml +++ b/.github/workflows/generate-index.yml @@ -3,7 +3,7 @@ name: Build and Deploy Python Index to GitHub Pages on: push: branches: - - main # Re-generate index on every change to the default branch + - main # Re-generate index on every change to the default branch paths: - artifacts/*.whl - generate-index.py @@ -13,9 +13,9 @@ on: # Minimal permissions required by the official Pages deployment action # NOTE: these are REPLACED, not merged, with job-level permissions permissions: - contents: read # to checkout the repository - pages: write # to deploy to Pages - id-token: write # to verify deployment origin + contents: read # to checkout the repository + pages: write # to deploy to Pages + id-token: write # to verify deployment origin concurrency: group: "deploy-pages" @@ -63,7 +63,7 @@ jobs: needs: build runs-on: ubuntu-latest environment: - name: github-pages # default Pages environment + name: github-pages # default Pages environment url: ${{ steps.deploy.outputs.page_url }} # Job-level permissions must explicitly include the same set permissions: From b85f9885bf12762b016082aa176471cad6ff601b Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Thu, 18 Sep 2025 19:33:14 +0300 Subject: [PATCH 31/33] tweak: Updated "broken filter" in run-tests.sh --- run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-tests.sh b/run-tests.sh index 662e834..4f8b6a0 100644 --- a/run-tests.sh +++ b/run-tests.sh @@ -26,7 +26,7 @@ for testfile in tests/*.py; do fi EXPECT_BROKEN=false - if [[ "$TEST_NAME" == *-broken.py ]]; then + if [[ "$TEST_NAME" == *\.broken.py ]]; then EXPECT_BROKEN=true fi From 2bcfe108fc9d02b5c54dfb913355b64bb4c5aae7 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Fri, 19 Sep 2025 10:14:37 +0300 Subject: [PATCH 32/33] feat: Added env var ALLOW_BROKEN to list broken tests --- testing/src/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/testing/src/main.py b/testing/src/main.py index a8bba0f..075f5f9 100644 --- a/testing/src/main.py +++ b/testing/src/main.py @@ -64,7 +64,12 @@ def _list_tests() -> list[str]: tests_dir = _resolve_tests_dir() pattern = os.path.join(tests_dir, "*.py") - files = [f for f in glob.glob(pattern) if ".skip" not in f and ".broken" not in f] + allow_broken = os.getenv("ALLOW_BROKEN") + files = [ + f + for f in glob.glob(pattern) + if ".skip" not in f and (".broken" not in f or allow_broken) + ] # Return just filenames for readability return [os.path.basename(f) for f in sorted(files)] From f889f33415c57026f4c8d1e5e229c0d424cee1a4 Mon Sep 17 00:00:00 2001 From: Lorentz Kinde Date: Fri, 19 Sep 2025 18:40:00 +0300 Subject: [PATCH 33/33] tweaks: Final things while push to prod + compile, tweaked readme --- README.md | 3 ++- testing/app.yaml | 5 +++++ testing/wasmer.toml | 22 ++++++---------------- 3 files changed, 13 insertions(+), 17 deletions(-) create mode 100644 testing/app.yaml diff --git a/README.md b/README.md index 44a4501..9342dea 100644 --- a/README.md +++ b/README.md @@ -294,6 +294,7 @@ Requirements: Run tests: +- `cd testing` - `uv pip compile pyproject.toml --python-version=3.13 --universal --extra-index-url https://pythonindex.wasix.org/simple --index-url=https://pypi.org/simple --emit-index-url --only-binary :all: -o wasmer-requirements.txt` - `uvx pip install -r wasmer-requirements.txt --target wasix-site-packages --platform wasix_wasm32 --only-binary=:all: --python-version=3.13 --compile` - `TEST_DIR=../tests/ wasmer run . --registry=wasmer.wtf --net --forward-host-env` @@ -303,7 +304,7 @@ This will run all tests via fastapi. You may also run each test individually by: -- `curl localhost:80801/list` +- `curl localhost:8081/list` - `curl localhost:8081/check/` This is needed when testing on edge, since `.../check` times out the workload. diff --git a/testing/app.yaml b/testing/app.yaml new file mode 100644 index 0000000..455e0bc --- /dev/null +++ b/testing/app.yaml @@ -0,0 +1,5 @@ +kind: wasmer.io/App.v0 +name: build-scripts +owner: lorentz-dev +package: . +app_id: da_2OPIqt7UeOLK diff --git a/testing/wasmer.toml b/testing/wasmer.toml index f4e5b25..e2efeb5 100644 --- a/testing/wasmer.toml +++ b/testing/wasmer.toml @@ -1,10 +1,10 @@ [package] -name = "wasmer/build-scripts" +name = "lorentz-dev/build-scripts" version = "0.1.10" entrypoint = "server" [dependencies] -"wasmer/python-native" = "=0.1.11" +"python/python" = "=3.13.1" [fs] "/app" = "./src" @@ -13,28 +13,18 @@ entrypoint = "server" [[command]] name = "server" -module = "wasmer/python-native:python" +module = "python/python:python" runner = "wasi" [command.annotations.wasi] main-args = ["/app/main.py"] -env = [ - "PYTHONEXECUTABLE=/bin/python", - "PYTHONHOME=/cpython", - "PYTHONPATH=/opt/venv/packages", - "HOME=/app", -] +env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "PYTHONPATH=/opt/venv/packages", "HOME=/app"] [[command]] name = "test" -module = "wasmer/python-native:python" +module = "python/python:python" runner = "wasi" [command.annotations.wasi] main-args = ["/app/run-tests.py"] -env = [ - "PYTHONEXECUTABLE=/bin/python", - "PYTHONHOME=/cpython", - "PYTHONPATH=/opt/venv/packages", - "HOME=/app", -] +env = ["PYTHONEXECUTABLE=/bin/python", "PYTHONHOME=/cpython", "PYTHONPATH=/opt/venv/packages", "HOME=/app"]