Investigate free-threaded Python (PEP 703) support

.github/workflows/tests.yml

@@ -14,6 +14,13 @@ jobs:
     name: "Run tests with Python ${{ matrix.python-version }}"
     runs-on: "ubuntu-latest"
 
+    # The suite takes about a minute; the slowest leg (PyPy) about three.
+    # Without a limit here a job that stops making progress -- a wedged
+    # thread in one of the concurrency tests, say -- is only stopped by
+    # the six-hour default, holding up the whole run and reporting
+    # nothing useful when it finally dies.
+    timeout-minutes: 20
+
     continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
@@ -25,13 +32,21 @@ jobs:
           - "3.12"
           - "3.13"
           - "3.14"
+          # Free-threaded (PEP 703) build.  Runs the same suite with the
+          # GIL disabled; see docs/free-threading.md.
+          - "3.14t"
         experimental: [ false ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
             fetch-depth: 0
-      - uses: "actions/setup-python@v4"
+      # setup-python must be >= v5.3: that is the first release that
+      # understands the free-threaded "t" suffix.  On v4, "3.14t" is
+      # looked up as a literal version against arch x64 rather than
+      # x64-freethreaded, and the job fails with "The version '3.14t'
+      # with architecture 'x64' was not found".
+      - uses: "actions/setup-python@v5"
         with:
           python-version: "${{ matrix.python-version }}"
           cache: "pip"

mkdocs.yml

@@ -39,6 +39,7 @@ nav:
       - Web app: example-webapp.md
     - Developing:
       - Contributing Guide: contributing.md
+      - Free-threaded Python: free-threading.md
   - References:
     - 'Mode':
       - mode.services: references/mode.services.md

mode/__init__.py

@@ -2,13 +2,11 @@
 
 __version__ = "0.0.1"
 
-import sys
 import typing
 from collections.abc import Mapping, Sequence
 
-# Lazy loading.
-# - See werkzeug/__init__.py for the rationale behind this.
-from types import ModuleType
+# Lazy loading, via the PEP 562 module __getattr__ defined at the bottom
+# of this file.
 from typing import Any
 
 # -eof meta-
@@ -33,6 +31,10 @@
     from .utils.times import Seconds, want_seconds
     from .worker import Worker
 
+# NOTE: Must list exactly the names in `all_by_module` below.  It is kept
+# as a literal (rather than derived) because ruff and mypy only recognise
+# the TYPE_CHECKING re-exports through a literal __all__; the sync is
+# enforced by tests/functional/test_thread_safety.py's lazy-import tests.
 __all__ = [
     "BaseSignal",
     "BaseSignalT",
@@ -88,47 +90,40 @@
         object_origins[item] = module
 
 
-class _module(ModuleType):
-    """Customized Python module."""
-
-    def __getattr__(self, name: str) -> Any:
-        if name in object_origins:
-            module = __import__(object_origins[name], None, None, [name])
-            for extra_name in all_by_module[module.__name__]:
-                setattr(self, extra_name, getattr(module, extra_name))
-            return getattr(module, name)
-        return ModuleType.__getattribute__(self, name)
-
-    def __dir__(self) -> Sequence[str]:
-        result = list(new_module.__all__)
-        result.extend(
-            (
-                "__file__",
-                "__path__",
-                "__doc__",
-                "__all__",
-                "__docformat__",
-                "__name__",
-                "__path__",
-                "VERSION",
-                "version_info",
-                "__package__",
-            )
-        )
-        return result
-
-
-# keep a reference to this module so that it's not garbage collected
-old_module = sys.modules[__name__]
-
-new_module = sys.modules[__name__] = _module(__name__)
-new_module.__dict__.update(
-    {
-        "__file__": __file__,
-        "__path__": __path__,
-        "__doc__": __doc__,
-        "__all__": tuple(object_origins),
-        "__version__": __version__,
-        "__package__": __package__,
-    }
-)
+# NOTE: This is a :pep:`562` module-level ``__getattr__``, and deliberately
+# *not* the older trick of defining a ``ModuleType`` subclass and swapping it
+# into ``sys.modules[__name__]`` at the end of this file.
+#
+# That swap was a race: it only happened once the module body had finished,
+# so a thread calling ``import mode`` while another thread was still
+# executing this file could be handed the original, pre-swap module object --
+# which has no ``__getattr__`` on it -- and every lazily-exported name below
+# raised ``AttributeError: module 'mode' has no attribute 'Service'``.  The
+# replacement module also carried no ``__spec__``, which denied the import
+# machinery the ``_initializing`` flag it uses to make the second thread wait.
+# Rare under the GIL, common on free-threaded (:pep:`703`) builds.
+#
+# A module ``__getattr__`` needs no swap at all, so the race cannot happen.
+def __getattr__(name: str) -> Any:
+    try:
+        origin = object_origins[name]
+    except KeyError:
+        raise AttributeError(
+            f"module {__name__!r} has no attribute {name!r}"
+        ) from None
+    module = __import__(origin, None, None, [name])
+    # Bind every name this module provides, not just the requested one, so
+    # that later lookups are plain globals and never reach __getattr__ again.
+    namespace = globals()
+    for extra_name in all_by_module[origin]:
+        namespace[extra_name] = getattr(module, extra_name)
+    return namespace[name]
+
+
+def __dir__() -> Sequence[str]:
+    # Everything actually in the module namespace, plus the lazy exports
+    # __getattr__ can still resolve.  Derived so it cannot drift into
+    # advertising names that do not exist (the old hand-written list was
+    # carried over from a template and promised VERSION/version_info,
+    # which no version of this module ever defined).
+    return sorted(set(globals()) | set(object_origins))

mode/locals.py

@@ -152,6 +152,28 @@ class XProxy(MutableMappingRole, AsyncContextManagerRole):
 PYPY = hasattr(sys, "pypy_version_info")
 SLOTS_ISSUE_PRESENT = sys.version_info < (3, 7)
 
+
+def _cooperative_init_subclass(cls: "type[Proxy[Any]]") -> None:
+    """Call the next ``__init_subclass__`` in ``Proxy``'s MRO.
+
+    This lives at module level, outside the class body, for one reason:
+    naming ``super`` inside a method of ``Proxy`` would make the compiler
+    add an implicit ``__class__`` closure cell to that class.  ``Proxy``
+    defines a ``__class__`` property, and on PyPy -- with a trace function
+    installed, i.e. under coverage -- a class body that has such a cell
+    resolves *every* mention of the name ``__class__`` to the cell rather
+    than to the class namespace.  Reading it then raises ``NameError:
+    name '__class__' is not defined`` at import time, and binding it
+    leaves no descriptor on the class at all, so proxies start reporting
+    themselves instead of the object they wrap.
+
+    Keeping the cell from existing keeps ``__class__`` an ordinary name in
+    that class body, which is what every interpreter has always handled.
+    Pinned by tests/unit/test_locals.py::test_Proxy_class_body_bytecode.
+    """
+    super(Proxy, cls).__init_subclass__()
+
+
 T = TypeVar("T")
 S = TypeVar("S")
 T_co = TypeVar("T_co", covariant=True)
@@ -199,7 +221,12 @@ class Proxy(Generic[T]):
         )
 
     def __init_subclass__(self, source: Optional[type[T]] = None) -> None:
-        super().__init_subclass__()
+        # NOTE: Delegated to a module-level helper on purpose -- do not
+        # inline this back to `super().__init_subclass__()`.  Naming `super`
+        # anywhere in this class body makes the compiler add an implicit
+        # `__class__` closure cell, which breaks the `__class__` property
+        # below on PyPy.  See `_cooperative_init_subclass`.
+        _cooperative_init_subclass(self)
         if source is not None:
             self._init_from_source(source)
         elif self.__proxy_source__ is not None:
@@ -283,6 +310,9 @@ def __doc__(self) -> Optional[str]:
     def _get_class(self) -> type[T]:
         return self._get_current_object().__class__
 
+    # NOTE: This ordinary property spelling is only safe while the class
+    # body has no implicit `__class__` closure cell -- see
+    # `_cooperative_init_subclass` before adding any use of `super` here.
     @property
     def __class__(self) -> Any:
         return self._get_class()

mode/loop/__init__.py

@@ -24,7 +24,18 @@
 mode.loop.use('eventlet')
 ```
 
-### gevent
+### gevent **deprecated, currently broken**
+
+!!! warning
+    This backend is unmaintained, has no test coverage, and does not
+    presently work on *any* interpreter: selecting it raises
+    `ImportError: Cannot import 'Loop' from mode.loop._gevent_loop` with
+    current gevent releases, on GIL-enabled and free-threaded builds
+    alike.  It also re-enables the GIL on free-threaded builds, because
+    `gevent.libev.corecext` does not declare that it is safe without it.
+
+    Selecting it raises a `DeprecationWarning`, and it will be removed in
+    a future major release.  Use `aio` (the default) or `uvloop`.
 
 Use [`gevent`](https://pypi.org/project/gevent) as the event loop.
 
@@ -59,6 +70,7 @@
 """
 
 import importlib
+import warnings
 from collections.abc import Mapping
 from typing import Optional
 
@@ -71,12 +83,34 @@
     "uvloop": "mode.loop.uvloop",
 }
 
+#: Backends that still resolve, but should not be used in new code.
+DEPRECATED_LOOPS: Mapping[str, str] = {
+    "gevent": (
+        "The gevent loop backend is deprecated and currently broken: it is "
+        "unmaintained, has no test coverage, and importing it fails with "
+        "current gevent releases on every interpreter (see "
+        "docs/free-threading.md).  It also re-enables the GIL on "
+        "free-threaded builds.  Use the 'aio' or 'uvloop' backend instead. "
+        "It will be removed in a future major release."
+    )
+}
+
 
 def use(loop: str) -> None:
     """Specify the event loop to use as a string.
 
     Loop must be one of: aio, eventlet, gevent, uvloop.
+
+    Note:
+        `gevent` is deprecated and currently broken -- selecting it raises
+        a `DeprecationWarning` and then fails to import.  See the module
+        docstring.
     """
+    deprecated = DEPRECATED_LOOPS.get(loop)
+    if deprecated is not None:
+        # stacklevel=2 attributes this to the caller, so it is actually
+        # shown when selected from an entrypoint module.
+        warnings.warn(deprecated, DeprecationWarning, stacklevel=2)
     mod = LOOPS.get(loop, loop)
     if mod is not None:
         importlib.import_module(mod)

mode/loop/gevent.py

@@ -1,12 +1,45 @@
-"""Enable [`gevent`](https://pypi.org/project/gevent) support for `asyncio`."""
+"""Enable [`gevent`](https://pypi.org/project/gevent) support for `asyncio`.
+
+!!! warning "Deprecated and currently broken"
+    This loop backend is unmaintained, has no test coverage, and does not
+    presently work on *any* interpreter -- importing it raises
+    `ImportError: Cannot import 'Loop' from mode.loop._gevent_loop` with
+    current gevent releases, on GIL-enabled and free-threaded builds
+    alike.  See `docs/free-threading.md` for the diagnosis.
+
+    Use the `aio` (default) or `uvloop` backend instead.  This module will
+    be removed in a future major release.
+"""
 
 import asyncio
 import os
+import sysconfig
 import warnings
 from typing import Optional, cast
 
 from mode.utils.loops import get_event_loop
 
+# NOTE: The DeprecationWarning for this backend is raised by
+# `mode.loop.use()`, not here.  A module-level `warnings.warn` is
+# attributed to whichever importlib frame executed the module body, and
+# DeprecationWarning is filtered out everywhere except `__main__` -- so it
+# would never be shown.  Raising it from `use()` with stacklevel=2 puts it
+# on the caller instead, which is where users select the backend.
+
+# NOTE: Deliberately the *build* flag, not `sys._is_gil_enabled()`.  The
+# runtime check would already read True by the time gevent has been
+# imported below, which is exactly the situation being reported.
+if sysconfig.get_config_var("Py_GIL_DISABLED"):
+    warnings.warn(
+        "The gevent loop is not usable on free-threaded builds: importing "
+        "gevent re-enables the GIL (gevent.libev.corecext does not declare "
+        "that it is safe without it), so selecting this loop silently gives "
+        "up free threading for the whole process.  Use the 'aio' or 'uvloop' "
+        "loop to keep the GIL disabled.",
+        RuntimeWarning,
+        stacklevel=2,
+    )
+
 os.environ["GEVENT_LOOP"] = "mode.loop._gevent_loop.Loop"
 try:
     import gevent