interpreters/python: Enable using pip to install Python packages

This commit enables using `pip` as a pre-compiled (pyc) built-in
distributed along with cpython.

Signed-off-by: Tiago Medicci <tiago.medicci@espressif.com>

interpreters/python/Kconfig

@@ -37,4 +37,21 @@ config INTERPRETERS_CPYTHON_PROGNAME
 	---help---
 		This is the name of the program that will be used from the nsh.
 
+config INTERPRETERS_CPYTHON_ENABLE_PIP
+	bool "Enable bundled pip"
+	default n
+	---help---
+		Enable bundling pip into the CPython module image. When enabled, the
+		build downloads the pip wheel and pre-installs it through a
+		site-packages .pth entry that points to ensurepip's bundled wheel.
+		Disable this to skip pip wheel download/integration entirely.
+
+
+config INTERPRETERS_CPYTHON_PYTHONPATH
+	string "CPython Python path"
+	default "/tmp"
+	---help---
+		This is the Python default search path for modules files. This is
+		required to be a writable path.
+
 endif

interpreters/python/Make.defs

@@ -27,6 +27,8 @@ CPYTHON_VERSION_MINOR=$(basename $(CPYTHON_VERSION))
 
 EXTRA_LIBPATHS += -L$(APPDIR)/interpreters/python/install/target
 EXTRA_LIBS += -lpython$(CPYTHON_VERSION_MINOR)
+EXTRA_LIBS += $(APPDIR)/interpreters/python/build/target/Modules/_hacl/libHacl_Hash_SHA2.a
+EXTRA_LIBS += $(APPDIR)/interpreters/python/build/target/Modules/expat/libexpat.a
 
 CONFIGURED_APPS += $(APPDIR)/interpreters/python
 endif

interpreters/python/Makefile

@@ -84,6 +84,10 @@ $(CPYTHON_UNPACKNAME): $(CPYTHON_ZIP)
 	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch
 	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0013-transform-functions-used-by-NuttX-to-lowercase.patch
 	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0014-insert-prefix-to-list_length-to-avoid-symbol-collisi.patch
+	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0015-keep-ensurepip-in-stdlib-archive.patch
+	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0016-fix-timezone-offset-check-when-time-t-is-unsigned.patch
+	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0017-stdlib-zip-keep-pydecimal-and-trim-tooling-extras.patch
+	$(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0018-ignore-chmod-on-nuttx-like-wasi.patch
 
 $(HOSTPYTHON):
 	mkdir -p $(HOSTBUILD)
@@ -92,6 +96,7 @@ $(HOSTPYTHON):
 			cd $(HOSTBUILD) && $(CPYTHON_PATH)/configure \
 			 --with-pydebug \
 			 --prefix=$(HOSTINSTALL) \
+			 --disable-test-modules \
 		 )
 	$(MAKE) -C $(HOSTBUILD) install
 
@@ -152,7 +157,7 @@ $(TARGETBUILD)/Makefile: $(HOSTPYTHON) $(CONFIG_SITE) $(SETUP_LOCAL)
 			AR="$(AR)" \
 			ARFLAGS=" " \
 			MACHDEP="$(MACHDEP)" \
-			OPT="-g -O0 -Wall" \
+			OPT="-O3" \
 			CONFIG_SITE="$(CONFIG_SITE)" \
 			$(CPYTHON_PATH)/configure \
 			 --prefix=${TARGETINSTALL} \
@@ -163,13 +168,44 @@ $(TARGETBUILD)/Makefile: $(HOSTPYTHON) $(CONFIG_SITE) $(SETUP_LOCAL)
 			 --without-mimalloc \
 			 --without-pymalloc \
 			 --disable-test-modules \
+			 --with-ensurepip=no \
 		)
+	$(Q) sed -i 's/^#define HAVE_LIBB2 1/\/* #undef HAVE_LIBB2 *\//' $(TARGETBUILD)/pyconfig.h
+	$(Q) sed -i 's/-lb2//g' $(TARGETBUILD)/Makefile
 
 $(TARGETLIBPYTHON): $(TARGETBUILD)/Makefile
+ifeq ($(CONFIG_INTERPRETERS_CPYTHON_ENABLE_PIP),y)
+	$(Q) mkdir -p $(CPYTHON_PATH)/Lib/ensurepip/_bundled
+	$(Q) ( \
+		PIP_WHEEL_VERSION=$$($(HOSTPYTHON) -c "import ensurepip; print(ensurepip._PIP_VERSION)"); \
+		PIP_WHEEL=$(CPYTHON_PATH)/Lib/ensurepip/_bundled/pip-$${PIP_WHEEL_VERSION}-py3-none-any.whl; \
+		if [ ! -f "$${PIP_WHEEL}" ]; then \
+			echo "Fetching pip wheel $${PIP_WHEEL_VERSION} for ensurepip bundle"; \
+			$(HOSTPYTHON) -m pip download --only-binary=:all: --no-deps --dest $(CPYTHON_PATH)/Lib/ensurepip/_bundled pip==$${PIP_WHEEL_VERSION}; \
+		fi; \
+		echo "Pre-compiling pip wheel with build Python (must match embedded CPython version)"; \
+		$(HOSTPYTHON) $(CURDIR)/repack_wheel_add_pyc.py "$${PIP_WHEEL}"; \
+	)
+endif
 	$(MAKE) -C $(TARGETBUILD) regen-frozen
 	$(MAKE) -C $(TARGETBUILD) libpython$(CPYTHON_VERSION_MINOR).a wasm_stdlib
 	$(Q) ( cp $(TARGETBUILD)/libpython$(CPYTHON_VERSION_MINOR).a $(TARGETLIBPYTHON) )
 	$(Q) $(UNPACK) $(TARGETMODULESPACK) -d $(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)
+ifeq ($(CONFIG_INTERPRETERS_CPYTHON_ENABLE_PIP),y)
+	$(Q) mkdir -p $(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)/site-packages
+	$(Q) ( \
+		set -e; \
+		BUNDLED_DIR=$(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)/ensurepip/_bundled; \
+		SITE_PACKAGES=$(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR)/site-packages; \
+		: > "$${SITE_PACKAGES}/bundled_wheels.pth"; \
+		for wheel in $${BUNDLED_DIR}/*.whl; do \
+			[ -f "$${wheel}" ] || continue; \
+			whl_name=$$(basename "$${wheel}"); \
+			echo "Pre-installing wheel via zipimport into target sys.path: $${whl_name}"; \
+			echo "../ensurepip/_bundled/$${whl_name}" >> "$${SITE_PACKAGES}/bundled_wheels.pth"; \
+		done; \
+	)
+endif
 
 MODULE    = $(CONFIG_INTERPRETERS_CPYTHON)
 

interpreters/python/Setup.local.in

@@ -4,7 +4,6 @@
 
 *disabled*
 _asyncio
-_blake2
 _bz2
 _codecs_cn
 _codecs_hk
@@ -15,19 +14,12 @@ _codecs_tw
 _ctypes
 _decimal
 _elementtree
-_hashlib
 _heapq
 _interpchannels
 _interpqueues
 _lsprof
 _lzma
-_md5
 _multibytecodec
-_sha1
-_sha2
-_sha2
-_sha3
-_sha3
 _sqlite3
 _ssl
 _statistics
@@ -41,9 +33,7 @@ _testlimitedcapi
 _uuid
 _xxtestfuzz
 _zoneinfo
-mmap
 pwd
-pyexpat
 readline
 resource
 xxsubtype

interpreters/python/config.site.in

@@ -21,4 +21,13 @@ export ac_cv_func_pipe="yes"
 export ac_cv_enable_strict_prototypes_warning="no"
 export ac_cv_func_getnameinfo="yes"
 export ac_cv_func_poll="yes"
-export ac_cv_func_gethostname="yes"
+export ac_cv_func_gethostname="yes"
+export ac_cv_func_lstat="yes"
+export ac_cv_func_readlink="yes"
+export ac_cv_func_realpath="yes"
+export ac_cv_func_getpid="yes"
+export ac_cv_func_utime="yes"
+export ac_cv_func_utimes="yes"
+export ac_cv_func_getuid="yes"
+export ac_cv_func_sysconf="yes"
+export ac_cv_func_umask="yes"

interpreters/python/patch/0015-keep-ensurepip-in-stdlib-archive.patch

@@ -0,0 +1,25 @@
+--- a/Tools/wasm/wasm_assets.py
++++ b/Tools/wasm/wasm_assets.py
+@@ -40,7 +40,6 @@ OMIT_FILES = (
+     # regression tests
+     "test/",
+     # package management
+-    "ensurepip/",
+     "venv/",
+     # other platforms
+     "_aix_support.py",
+@@ -148,6 +147,13 @@ def create_stdlib_zip(
+             if entry.name.endswith(".py") or entry.is_dir():
+                 # writepy() writes .pyc files (bytecode).
+                 pzf.writepy(entry, filterfunc=filterfunc)
++
++        # Preserve ensurepip wheel payloads so `python -m ensurepip` can
++        # bootstrap pip on targets that consume this stdlib zip archive.
++        bundled_wheels = args.srcdir_lib / "ensurepip" / "_bundled"
++        if bundled_wheels.is_dir():
++            for wheel in sorted(bundled_wheels.glob("*.whl")):
++                pzf.write(wheel, arcname=f"ensurepip/_bundled/{wheel.name}")
+ 
+ 
+ def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]:
+

interpreters/python/patch/0016-fix-timezone-offset-check-when-time-t-is-unsigned.patch

@@ -0,0 +1,21 @@
+--- a/Modules/timemodule.c
++++ b/Modules/timemodule.c
+@@ -1800,15 +1800,15 @@ static int
+     static const time_t YEAR = (365 * 24 + 6) * 3600;
+     time_t t;
+     struct tm p;
+-    time_t janzone_t, julyzone_t;
++    long long janzone_t, julyzone_t;
+     char janname[10], julyname[10];
+     t = (time((time_t *)0) / YEAR) * YEAR;
+     _PyTime_localtime(t, &p);
+     get_zone(janname, 9, &p);
+-    janzone_t = -get_gmtoff(t, &p);
++    janzone_t = -(long long)get_gmtoff(t, &p);
+     janname[9] = '\0';
+     t += YEAR/2;
+     _PyTime_localtime(t, &p);
+     get_zone(julyname, 9, &p);
+-    julyzone_t = -get_gmtoff(t, &p);
++    julyzone_t = -(long long)get_gmtoff(t, &p);
+     julyname[9] = '\0';

interpreters/python/patch/0017-stdlib-zip-keep-pydecimal-and-trim-tooling-extras.patch

@@ -0,0 +1,35 @@
+--- a/Tools/wasm/wasm_assets.py
++++ b/Tools/wasm/wasm_assets.py
+@@ -47,13 +47,20 @@
+     # webbrowser
+     "antigravity.py",
+     "webbrowser.py",
+-    # Pure Python implementations of C extensions
+-    "_pydecimal.py",
++    # Pure Python implementations of C extensions.
++    # NOTE: keep "_pydecimal.py" so decimal.py can fall back to it when the
++    # _decimal C extension is not built (NuttX targets do not link libmpdec).
+     "_pyio.py",
+     # concurrent threading
+     "concurrent/futures/thread.py",
+     # Misc unused or large files
+     "pydoc_data/",
++    # Tooling/REPL extras not needed on a constrained embedded target.
++    "unittest/",
++    "_pyrepl/",
++    "idlelib/",
++    "turtledemo/",
++    "wsgiref/",
+ )
+ 
+ # Synchronous network I/O and protocols are not supported; for example,
+@@ -80,7 +87,8 @@
+     "_asyncio": ["asyncio/"],
+     "_curses": ["curses/"],
+     "_ctypes": ["ctypes/"],
+-    "_decimal": ["decimal.py"],
++    # decimal.py is intentionally NOT omitted here: it ships a pure-Python
++    # fallback (_pydecimal) used when the _decimal C ext is unavailable.
+     "_dbm": ["dbm/ndbm.py"],
+     "_gdbm": ["dbm/gnu.py"],
+     "_json": ["json/"],

interpreters/python/patch/0018-ignore-chmod-on-nuttx-like-wasi.patch

@@ -0,0 +1,25 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Tiago Medicci <tiago.medicci@espressif.com>
+Date: Thu, 7 May 2026 14:00:00 -0300
+Subject: [PATCH] posixmodule: ignore chmod on NuttX like WASI
+
+NuttX's tmpfs does not implement chstat, so chmod fails with ENOSYS.
+Apply the same workaround already used for WASI: silently succeed
+when HAVE_CHMOD is not defined.
+
+---
+ Modules/posixmodule.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+--- a/Modules/posixmodule.c
++++ b/Modules/posixmodule.c
+@@ -3608,8 +3608,8 @@
+ #ifdef HAVE_CHMOD
+         result = chmod(path->narrow, mode);
+-#elif defined(__wasi__)
+-        // WASI SDK 15.0 does not support chmod.
++#elif defined(__wasi__) || defined(__NuttX__)
++        // WASI SDK 15.0 and NuttX do not fully support chmod.
+         // Ignore missing syscall for now.
+         result = 0;
+ #else

interpreters/python/python_wrapper.c

@@ -198,5 +198,7 @@ int main(int argc, FAR char *argv[])
 
   setenv("PYTHON_BASIC_REPL", "1", 1);
 
+  setenv("PYTHONPATH", CONFIG_INTERPRETERS_CPYTHON_PYTHONPATH, 1);
+
   return py_bytesmain(argc, argv);
 }

interpreters/python/repack_wheel_add_pyc.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+#
+# Repack a wheel in-place: compile pip/*.py to legacy sibling *.pyc (compileall
+# -b: required for zipimport, which does not read PEP 3147 __pycache__/ names),
+# remove the .py sources, and rewrite *.dist-info/RECORD.
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import shutil
+import subprocess
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+
+
+def wheel_record_hash(data: bytes) -> str:
+    digest = hashlib.sha256(data).digest()
+    return base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=")
+
+
+def wheel_has_pip_py_sources(zf: zipfile.ZipFile) -> bool:
+    return any(n.startswith("pip/") and n.endswith(".py") for n in zf.namelist())
+
+
+def wheel_has_legacy_pip_bytecode(zf: zipfile.ZipFile) -> bool:
+    return "pip/__init__.pyc" in zf.namelist()
+
+
+def strip_pip_py_sources(pip_dir: Path) -> int:
+    """Remove pip/**/*.py after sibling legacy *.pyc exists (compileall -b output)."""
+    removed = 0
+    for path in sorted(pip_dir.rglob("*.py")):
+        if not path.is_file():
+            continue
+        pyc = path.with_suffix(".pyc")
+        if not pyc.is_file():
+            rel = path.relative_to(pip_dir)
+            raise SystemExit(
+                f"missing legacy .pyc for pip/{rel.as_posix()}, refusing to delete source"
+            )
+        path.unlink()
+        removed += 1
+    return removed
+
+
+def rebuild_record(root: Path) -> None:
+    dist_infos = sorted(root.glob("*.dist-info"))
+    if len(dist_infos) != 1:
+        raise SystemExit(
+            f"expected one *.dist-info, got {[p.name for p in dist_infos]}"
+        )
+    di = dist_infos[0]
+    record_path = di / "RECORD"
+    record_rel = f"{di.name}/RECORD"
+    lines: list[str] = []
+    for path in sorted(root.rglob("*")):
+        if not path.is_file():
+            continue
+        rel = path.relative_to(root).as_posix()
+        if rel == record_rel:
+            continue
+        body = path.read_bytes()
+        lines.append(f"{rel},sha256={wheel_record_hash(body)},{len(body)}")
+    lines.append(f"{record_rel},,")
+    record_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def repack(whl_path: Path, *, force: bool) -> None:
+    whl_path = whl_path.resolve()
+    if not whl_path.is_file():
+        raise SystemExit(f"missing wheel: {whl_path}")
+
+    with zipfile.ZipFile(whl_path) as zf:
+        has_py = wheel_has_pip_py_sources(zf)
+        if not has_py:
+            if not wheel_has_legacy_pip_bytecode(zf):
+                raise SystemExit(
+                    "repack_wheel_add_pyc: wheel has no pip/*.py and no pip/__init__.pyc "
+                    "(corrupt or old tool output). Delete the bundled pip-*.whl and rebuild."
+                )
+            if not force:
+                print(
+                    f"repack_wheel_add_pyc: skip (pip already bytecode-only): {whl_path.name}"
+                )
+                return
+
+    tmpdir = tempfile.mkdtemp(prefix="pip-whl-pyc-")
+    try:
+        root = Path(tmpdir)
+        with zipfile.ZipFile(whl_path) as zf:
+            zf.extractall(root)
+
+        pip_dir = root / "pip"
+        if not pip_dir.is_dir():
+            raise SystemExit("wheel has no pip/ top-level package")
+
+        subprocess.run(
+            [sys.executable, "-m", "compileall", "-q", "-f", "-b", str(pip_dir)],
+            cwd=str(root),
+            check=True,
+        )
+        n_py = strip_pip_py_sources(pip_dir)
+        rebuild_record(root)
+
+        out_path = whl_path.with_suffix(whl_path.suffix + ".tmp")
+        with zipfile.ZipFile(out_path, "w", compression=zipfile.ZIP_DEFLATED) as out:
+            for path in sorted(root.rglob("*")):
+                if path.is_file():
+                    arcname = path.relative_to(root).as_posix()
+                    out.write(path, arcname)
+
+        out_path.replace(whl_path)
+        print(
+            f"repack_wheel_add_pyc: bytecode-only pip ({n_py} .py removed) -> {whl_path.name}"
+        )
+    finally:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("wheel", type=Path, help="path to .whl (updated in place)")
+    ap.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        help="repack even if pip is already .pyc-only",
+    )
+    args = ap.parse_args()
+    repack(args.wheel, force=args.force)
+
+
+if __name__ == "__main__":
+    main()