From ab041b13fccf3342d9b31101a7767a67966765e9 Mon Sep 17 00:00:00 2001 From: Logan Snow Date: Mon, 30 Mar 2026 12:02:37 -0400 Subject: [PATCH 1/7] Add PY_FORCE_LAZY_IMPORTS env var for deferred module loading Implements lazy imports inspired by PEP 690 and HRT's CPython fork. When PY_FORCE_LAZY_IMPORTS=true is set, top-level `import foo` statements are deferred until the module's attributes are first accessed. This can significantly reduce startup time for applications with heavy imports. - Lib/_lazy_imports.py: LazyModule proxy type and __import__ hook - Lib/site.py: activate lazy imports based on env var at startup - Lib/test/test_lazy_imports.py: 15 unit and integration tests Co-Authored-By: Claude Opus 4.6 (1M context) --- Lib/_lazy_imports.py | 202 ++++++++++++++++++++++++++++++++++ Lib/site.py | 8 ++ Lib/test/test_lazy_imports.py | 202 ++++++++++++++++++++++++++++++++++ 3 files changed, 412 insertions(+) create mode 100644 Lib/_lazy_imports.py create mode 100644 Lib/test/test_lazy_imports.py diff --git a/Lib/_lazy_imports.py b/Lib/_lazy_imports.py new file mode 100644 index 00000000000000..81f40fbe7558d9 --- /dev/null +++ b/Lib/_lazy_imports.py @@ -0,0 +1,202 @@ +"""Lazy import machinery for CPython. + +When enabled via the PY_FORCE_LAZY_IMPORTS environment variable, top-level +``import foo`` statements are deferred: the module is not actually loaded +until one of its attributes is first accessed. ``from foo import bar`` +statements remain eager because the IMPORT_FROM opcode needs to resolve +the attribute immediately. + +This is a simplified implementation inspired by PEP 690 and the approach +described by Hudson River Trading. + +Usage:: + + import _lazy_imports + _lazy_imports.install() # replaces builtins.__import__ + _lazy_imports.uninstall() # restores the original __import__ +""" + +import builtins +import sys +import types + +__all__ = ["install", "uninstall", "is_enabled", "LazyModule"] + +_original_import = None + +# Modules that must never be imported lazily. These are either critical +# to the import system itself or have load-time side effects that many +# other modules depend on. +_EXCLUDED = frozenset({ + "importlib", "importlib._bootstrap", "importlib._bootstrap_external", + "importlib.abc", "importlib.machinery", "importlib.util", + "_frozen_importlib", "_frozen_importlib_external", + "_imp", "_thread", "_weakref", "_io", + "sys", "builtins", "marshal", "posix", "nt", "winreg", + "errno", "os", "os.path", "posixpath", "ntpath", + "_signal", "signal", + "site", "_sitebuiltins", + "_lazy_imports", + # encodings are needed during startup + "encodings", "codecs", "_codecs", + # warnings infrastructure + "warnings", "_warnings", + # abc machinery + "abc", "_abc", + # threading used by import lock + "threading", "_threading", +}) + + +class LazyModule(types.ModuleType): + """A module subclass that defers the actual import until first attribute access. + + When an attribute is looked up on a ``LazyModule``, the real module is + loaded via the original ``__import__``, the ``LazyModule`` entry in + ``sys.modules`` is replaced with the real module, and the attribute + value is returned transparently. + """ + + # Use slots to keep lazy bookkeeping separate from normal module attrs. + # The names are mangled with a prefix unlikely to collide. + __slots__ = ( + "_lazy_name", + "_lazy_full_name", + "_lazy_real", + "_lazy_loading", + ) + + def __new__(cls, name, full_name=None): + self = types.ModuleType.__new__(cls, name) + object.__setattr__(self, "_lazy_name", name) + object.__setattr__(self, "_lazy_full_name", full_name or name) + object.__setattr__(self, "_lazy_real", None) + object.__setattr__(self, "_lazy_loading", False) + return self + + # ------------------------------------------------------------------ + # Resolution + # ------------------------------------------------------------------ + + def _resolve(self): + """Load the real module, replace ourselves in sys.modules, return it.""" + real = object.__getattribute__(self, "_lazy_real") + if real is not None: + return real + if object.__getattribute__(self, "_lazy_loading"): + # Re-entrant call while we are already loading -- fall through + # to avoid infinite recursion. + return self + object.__setattr__(self, "_lazy_loading", True) + name = object.__getattribute__(self, "_lazy_name") + full_name = object.__getattribute__(self, "_lazy_full_name") + try: + # Remove ourselves from sys.modules so the real import machinery + # does not short-circuit back to us. + sys.modules.pop(name, None) + _original_import(full_name) + real = sys.modules[name] + object.__setattr__(self, "_lazy_real", real) + return real + except Exception: + # If the import fails, put ourselves back so the next attempt + # can retry. + sys.modules.setdefault(name, self) + raise + finally: + object.__setattr__(self, "_lazy_loading", False) + + # ------------------------------------------------------------------ + # Proxy dunder methods + # ------------------------------------------------------------------ + + def __getattr__(self, attr): + return getattr(self._resolve(), attr) + + def __repr__(self): + real = object.__getattribute__(self, "_lazy_real") + if real is not None: + return repr(real) + name = object.__getattribute__(self, "_lazy_name") + return f"" + + def __dir__(self): + return dir(self._resolve()) + + +# ------------------------------------------------------------------ +# Import hook +# ------------------------------------------------------------------ + +def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): + """Replacement for builtins.__import__ that defers simple imports.""" + # Relative imports -- always eager. + if level > 0: + return _original_import(name, globals, locals, fromlist, level) + + # ``from X import Y`` -- eager, because IMPORT_FROM needs real attrs. + if fromlist: + # If the module is currently a lazy stub, resolve it first so the + # real import machinery sees the real module (or nothing). + mod = sys.modules.get(name) + if isinstance(mod, LazyModule): + mod._resolve() + return _original_import(name, globals, locals, fromlist, level) + + # Already in sys.modules -- return as-is (may be lazy or real). + mod = sys.modules.get(name) + if mod is not None: + # For dotted imports (import a.b.c), __import__ must return + # the top-level package. If 'name' has dots, get top-level. + top = name.partition(".")[0] + if top != name: + top_mod = sys.modules.get(top) + if top_mod is not None: + return top_mod + return mod + + # Excluded modules -- always eager. + if name in _EXCLUDED: + return _original_import(name, globals, locals, fromlist, level) + + # Dotted import: ``import a.b.c`` -- create a lazy stub for the + # top-level package only. When resolved, it will import the full + # dotted path. + top = name.partition(".")[0] + if top != name: + if top not in sys.modules: + lazy = LazyModule(top, full_name=name) + sys.modules[top] = lazy + return sys.modules[top] + + # Simple ``import foo`` -- create a lazy stub. + lazy = LazyModule(name) + sys.modules[name] = lazy + return lazy + + +# ------------------------------------------------------------------ +# Public API +# ------------------------------------------------------------------ + +def install(): + """Replace ``builtins.__import__`` with the lazy import hook.""" + global _original_import + if _original_import is not None: + return # already installed + _original_import = builtins.__import__ + builtins.__import__ = _lazy_import_hook + + +def uninstall(): + """Restore the original ``builtins.__import__``.""" + global _original_import + if _original_import is None: + return + builtins.__import__ = _original_import + _original_import = None + + +def is_enabled(): + """Return True if the lazy import hook is currently active.""" + return _original_import is not None diff --git a/Lib/site.py b/Lib/site.py index 041dca113a572e..e6e5f221e3273b 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -681,6 +681,13 @@ def execusercustomize(): (err.__class__.__name__, err)) +def _maybe_enable_lazy_imports(): + """Enable lazy imports if the PY_FORCE_LAZY_IMPORTS env var is set.""" + if os.environ.get("PY_FORCE_LAZY_IMPORTS", "").lower() in ("1", "true", "yes"): + import _lazy_imports + _lazy_imports.install() + + def main(): """Add standard site-specific directories to the module search path. @@ -706,6 +713,7 @@ def main(): sethelper() if not sys.flags.isolated: enablerlcompleter() + _maybe_enable_lazy_imports() execsitecustomize() if ENABLE_USER_SITE: execusercustomize() diff --git a/Lib/test/test_lazy_imports.py b/Lib/test/test_lazy_imports.py new file mode 100644 index 00000000000000..ed54425800f41d --- /dev/null +++ b/Lib/test/test_lazy_imports.py @@ -0,0 +1,202 @@ +"""Tests for the _lazy_imports module.""" + +import sys +import os +import types +import unittest +import subprocess + + +class TestLazyModule(unittest.TestCase): + """Unit tests for the LazyModule class and install/uninstall API.""" + + def setUp(self): + import _lazy_imports + self._lazy_imports = _lazy_imports + # Save original state. + self._orig_modules = sys.modules.copy() + self._lazy_imports.install() + + def tearDown(self): + self._lazy_imports.uninstall() + # Restore sys.modules to avoid leaking state between tests. + sys.modules.clear() + sys.modules.update(self._orig_modules) + + def test_import_creates_lazy_module(self): + """A plain ``import`` of an unloaded module creates a LazyModule stub.""" + # Pick a stdlib module unlikely to be loaded yet in a fresh test. + sys.modules.pop("decimal", None) + import decimal + self.assertIsInstance(decimal, self._lazy_imports.LazyModule) + + def test_lazy_module_resolves_on_attr_access(self): + """Accessing an attribute on a LazyModule triggers real loading.""" + sys.modules.pop("textwrap", None) + import textwrap + self.assertIsInstance(textwrap, self._lazy_imports.LazyModule) + # This attribute access should trigger the real import. + result = textwrap.dedent(" hello") + self.assertEqual(result, "hello") + # After resolution, sys.modules should hold the real module. + self.assertNotIsInstance(sys.modules["textwrap"], + self._lazy_imports.LazyModule) + + def test_from_import_is_eager(self): + """``from X import Y`` must work and be resolved eagerly.""" + sys.modules.pop("base64", None) + from base64 import b64encode + self.assertTrue(callable(b64encode)) + # The module itself should have been loaded (not lazy). + self.assertNotIsInstance(sys.modules.get("base64"), + self._lazy_imports.LazyModule) + + def test_already_loaded_module_is_returned_directly(self): + """If a module is already in sys.modules, return it as-is.""" + import json # force eager + json_mod = sys.modules["json"] + import json as json2 + self.assertIs(json2, json_mod) + + def test_excluded_module_is_not_lazy(self): + """Modules in the exclusion set are imported eagerly.""" + sys.modules.pop("os", None) + import os as os_mod + self.assertNotIsInstance(os_mod, self._lazy_imports.LazyModule) + + def test_install_uninstall(self): + """install() and uninstall() toggle the hook correctly.""" + self.assertTrue(self._lazy_imports.is_enabled()) + self._lazy_imports.uninstall() + self.assertFalse(self._lazy_imports.is_enabled()) + self._lazy_imports.install() + self.assertTrue(self._lazy_imports.is_enabled()) + + def test_lazy_module_repr(self): + """LazyModule repr is informative before and after resolution.""" + sys.modules.pop("decimal", None) + import decimal + self.assertIn("lazy module", repr(decimal)) + self.assertIn("decimal", repr(decimal)) + # Resolve it. + decimal.Decimal + self.assertNotIn("lazy module", repr(decimal)) + + def test_lazy_module_dir(self): + """dir() on a LazyModule resolves and returns the real dir.""" + sys.modules.pop("textwrap", None) + import textwrap + d = dir(textwrap) + self.assertIn("dedent", d) + + def test_import_error_propagates(self): + """Importing a nonexistent module raises ImportError on access.""" + sys.modules.pop("nonexistent_module_xyz_123", None) + import nonexistent_module_xyz_123 as mod + self.assertIsInstance(mod, self._lazy_imports.LazyModule) + with self.assertRaises(ImportError): + mod.some_attr + + +class TestEnvironmentVariable(unittest.TestCase): + """Integration tests that verify the PY_FORCE_LAZY_IMPORTS env var works + end-to-end via subprocess.""" + + def _run_python(self, code, env_val=None): + env = os.environ.copy() + if env_val is not None: + env["PY_FORCE_LAZY_IMPORTS"] = env_val + else: + env.pop("PY_FORCE_LAZY_IMPORTS", None) + return subprocess.run( + [sys.executable, "-c", code], + capture_output=True, text=True, env=env, + timeout=30, + ) + + def test_env_var_enables_lazy_imports(self): + """With PY_FORCE_LAZY_IMPORTS=true, imports are lazy.""" + code = ( + "import sys\n" + "import _lazy_imports\n" + "print(_lazy_imports.is_enabled())\n" + ) + result = self._run_python(code, env_val="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "True") + + def test_env_var_disabled_by_default(self): + """Without PY_FORCE_LAZY_IMPORTS, imports are not lazy.""" + code = ( + "import _lazy_imports\n" + "print(_lazy_imports.is_enabled())\n" + ) + result = self._run_python(code, env_val=None) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "False") + + def test_lazy_import_defers_loading(self): + """Prove that a module is NOT in sys.modules until accessed.""" + code = ( + "import sys\n" + # decimal should not be loaded at startup with lazy imports + "# Now import decimal lazily\n" + "import decimal\n" + "loaded_before = 'decimal' in sys.modules and " + "not hasattr(sys.modules['decimal'], '_lazy_name')\n" + "print('before:', loaded_before)\n" + "# Now trigger the actual load\n" + "decimal.Decimal('1.0')\n" + "loaded_after = 'decimal' in sys.modules and " + "not hasattr(sys.modules['decimal'], '_lazy_name')\n" + "print('after:', loaded_after)\n" + ) + result = self._run_python(code, env_val="1") + self.assertEqual(result.returncode, 0, result.stderr) + lines = result.stdout.strip().splitlines() + self.assertEqual(lines[0], "before: False", + "Module should not be really loaded before access") + self.assertEqual(lines[1], "after: True", + "Module should be really loaded after access") + + def test_from_import_works_with_lazy(self): + """``from X import Y`` works correctly under lazy imports.""" + code = ( + "from base64 import b64encode\n" + "print(b64encode(b'hello').decode())\n" + ) + result = self._run_python(code, env_val="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "aGVsbG8=") + + def test_lazy_import_functional(self): + """A lazily imported module works correctly end-to-end.""" + code = ( + "import json\n" + "result = json.dumps({'key': 'value'})\n" + "print(result)\n" + ) + result = self._run_python(code, env_val="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), '{"key": "value"}') + + def test_env_var_accepts_multiple_values(self): + """PY_FORCE_LAZY_IMPORTS accepts '1', 'true', 'yes' (case insensitive).""" + code = "import _lazy_imports; print(_lazy_imports.is_enabled())" + for val in ("1", "true", "True", "TRUE", "yes", "Yes"): + with self.subTest(val=val): + result = self._run_python(code, env_val=val) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "True", + f"PY_FORCE_LAZY_IMPORTS={val} should enable lazy imports") + + for val in ("0", "false", "no", ""): + with self.subTest(val=val): + result = self._run_python(code, env_val=val) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "False", + f"PY_FORCE_LAZY_IMPORTS={val} should NOT enable lazy imports") + + +if __name__ == "__main__": + unittest.main() From f781aa632f32eac0cb5ca81590c494ab90cfba39 Mon Sep 17 00:00:00 2001 From: Logan Snow Date: Mon, 30 Mar 2026 13:31:13 -0400 Subject: [PATCH 2/7] Fix LazyModule TypeError with full_name keyword argument Switch from __new__ to __init__ so the full_name kwarg is consumed by LazyModule rather than leaking to ModuleType.__init__. Co-Authored-By: Claude Opus 4.6 (1M context) --- Lib/_lazy_imports.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/_lazy_imports.py b/Lib/_lazy_imports.py index 81f40fbe7558d9..31ba5e97c04f63 100644 --- a/Lib/_lazy_imports.py +++ b/Lib/_lazy_imports.py @@ -66,13 +66,12 @@ class LazyModule(types.ModuleType): "_lazy_loading", ) - def __new__(cls, name, full_name=None): - self = types.ModuleType.__new__(cls, name) + def __init__(self, name, full_name=None): + super().__init__(name) object.__setattr__(self, "_lazy_name", name) object.__setattr__(self, "_lazy_full_name", full_name or name) object.__setattr__(self, "_lazy_real", None) object.__setattr__(self, "_lazy_loading", False) - return self # ------------------------------------------------------------------ # Resolution From 8a146ce7588e0a47d74a3a21871dbb1df0a5a973 Mon Sep 17 00:00:00 2001 From: Logan Snow Date: Mon, 30 Mar 2026 14:26:59 -0400 Subject: [PATCH 3/7] Fix lazy imports: make dotted imports eager Dotted imports (import a.b.c) must be eager because they need to actually load submodules as a side effect. Previously the hook would skip loading submodules when the top-level package was already in sys.modules, breaking code like `import email.mime.text` where internal submodule imports rely on sibling submodules being loaded. Co-Authored-By: Claude Opus 4.6 (1M context) --- Lib/_lazy_imports.py | 37 +++++++++++++---------------------- Lib/test/test_lazy_imports.py | 25 +++++++++++++++++++++++ 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/Lib/_lazy_imports.py b/Lib/_lazy_imports.py index 31ba5e97c04f63..875c7dccd181c0 100644 --- a/Lib/_lazy_imports.py +++ b/Lib/_lazy_imports.py @@ -58,18 +58,15 @@ class LazyModule(types.ModuleType): """ # Use slots to keep lazy bookkeeping separate from normal module attrs. - # The names are mangled with a prefix unlikely to collide. __slots__ = ( "_lazy_name", - "_lazy_full_name", "_lazy_real", "_lazy_loading", ) - def __init__(self, name, full_name=None): + def __init__(self, name): super().__init__(name) object.__setattr__(self, "_lazy_name", name) - object.__setattr__(self, "_lazy_full_name", full_name or name) object.__setattr__(self, "_lazy_real", None) object.__setattr__(self, "_lazy_loading", False) @@ -88,12 +85,11 @@ def _resolve(self): return self object.__setattr__(self, "_lazy_loading", True) name = object.__getattribute__(self, "_lazy_name") - full_name = object.__getattribute__(self, "_lazy_full_name") try: # Remove ourselves from sys.modules so the real import machinery # does not short-circuit back to us. sys.modules.pop(name, None) - _original_import(full_name) + _original_import(name) real = sys.modules[name] object.__setattr__(self, "_lazy_real", real) return real @@ -142,32 +138,27 @@ def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): mod._resolve() return _original_import(name, globals, locals, fromlist, level) + # Dotted imports (import a.b.c) -- always eager. These must actually + # load submodules as a side effect; deferring them breaks code that + # relies on ``a.b`` being populated after ``import a.b.c``. + if "." in name: + # Resolve any lazy stub for the top-level package first so the + # real import machinery can traverse its __path__. + top = name.partition(".")[0] + mod = sys.modules.get(top) + if isinstance(mod, LazyModule): + mod._resolve() + return _original_import(name, globals, locals, fromlist, level) + # Already in sys.modules -- return as-is (may be lazy or real). mod = sys.modules.get(name) if mod is not None: - # For dotted imports (import a.b.c), __import__ must return - # the top-level package. If 'name' has dots, get top-level. - top = name.partition(".")[0] - if top != name: - top_mod = sys.modules.get(top) - if top_mod is not None: - return top_mod return mod # Excluded modules -- always eager. if name in _EXCLUDED: return _original_import(name, globals, locals, fromlist, level) - # Dotted import: ``import a.b.c`` -- create a lazy stub for the - # top-level package only. When resolved, it will import the full - # dotted path. - top = name.partition(".")[0] - if top != name: - if top not in sys.modules: - lazy = LazyModule(top, full_name=name) - sys.modules[top] = lazy - return sys.modules[top] - # Simple ``import foo`` -- create a lazy stub. lazy = LazyModule(name) sys.modules[name] = lazy diff --git a/Lib/test/test_lazy_imports.py b/Lib/test/test_lazy_imports.py index ed54425800f41d..75480446ee436b 100644 --- a/Lib/test/test_lazy_imports.py +++ b/Lib/test/test_lazy_imports.py @@ -97,6 +97,20 @@ def test_import_error_propagates(self): with self.assertRaises(ImportError): mod.some_attr + def test_dotted_import_is_eager(self): + """Dotted imports like ``import a.b`` are always eager.""" + sys.modules.pop("email", None) + sys.modules.pop("email.mime", None) + sys.modules.pop("email.mime.text", None) + import email.mime.text + # The submodule must actually be loaded (not lazy). + self.assertNotIsInstance(sys.modules.get("email"), + self._lazy_imports.LazyModule) + self.assertNotIsInstance(sys.modules.get("email.mime.text"), + self._lazy_imports.LazyModule) + # And the submodule must be accessible. + self.assertTrue(hasattr(email.mime.text, "MIMEText")) + class TestEnvironmentVariable(unittest.TestCase): """Integration tests that verify the PY_FORCE_LAZY_IMPORTS env var works @@ -180,6 +194,17 @@ def test_lazy_import_functional(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout.strip(), '{"key": "value"}') + def test_dotted_import_works_with_lazy(self): + """Dotted imports work correctly under lazy imports.""" + code = ( + "import email.mime.text\n" + "msg = email.mime.text.MIMEText('hello')\n" + "print(type(msg).__name__)\n" + ) + result = self._run_python(code, env_val="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "MIMEText") + def test_env_var_accepts_multiple_values(self): """PY_FORCE_LAZY_IMPORTS accepts '1', 'true', 'yes' (case insensitive).""" code = "import _lazy_imports; print(_lazy_imports.is_enabled())" From 6d4e9fc6aa1f403b6d84c8da8329ad88ff503d78 Mon Sep 17 00:00:00 2001 From: Logan Snow Date: Mon, 30 Mar 2026 14:30:12 -0400 Subject: [PATCH 4/7] Fix LazyModule: use __getattribute__ to properly proxy all attrs Use __getattribute__ instead of __getattr__ so that attributes which exist on the LazyModule itself (like __dict__, __name__, __spec__) are properly delegated to the real module after resolution. This fixes ssl, enum._convert_, and anything else that inspects module.__dict__ on a lazily-imported module. Co-Authored-By: Claude Opus 4.6 (1M context) --- Lib/_lazy_imports.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/Lib/_lazy_imports.py b/Lib/_lazy_imports.py index 875c7dccd181c0..cbe3f14fbae1df 100644 --- a/Lib/_lazy_imports.py +++ b/Lib/_lazy_imports.py @@ -105,8 +105,27 @@ def _resolve(self): # Proxy dunder methods # ------------------------------------------------------------------ - def __getattr__(self, attr): - return getattr(self._resolve(), attr) + # We need __getattribute__ (not just __getattr__) because attributes + # like __dict__, __name__, __spec__ etc. exist on the LazyModule + # itself (set by ModuleType.__init__). Using only __getattr__ would + # let those stale values leak out instead of delegating to the real + # module -- this breaks e.g. enum._convert_(source=lazy_mod) which + # iterates source.__dict__. + + _INTERNAL_ATTRS = frozenset({ + "_lazy_name", "_lazy_real", "_lazy_loading", "_resolve", + "__class__", + }) + + def __getattribute__(self, attr): + # Fast path: internal bookkeeping attrs must not trigger resolution. + if attr in LazyModule._INTERNAL_ATTRS: + return object.__getattribute__(self, attr) + real = object.__getattribute__(self, "_lazy_real") + if real is not None: + return getattr(real, attr) + # Not yet resolved -- resolve now. + return getattr(object.__getattribute__(self, "_resolve")(), attr) def __repr__(self): real = object.__getattribute__(self, "_lazy_real") @@ -116,7 +135,10 @@ def __repr__(self): return f"" def __dir__(self): - return dir(self._resolve()) + real = object.__getattribute__(self, "_lazy_real") + if real is not None: + return dir(real) + return dir(object.__getattribute__(self, "_resolve")()) # ------------------------------------------------------------------ From 9e165e877802eac87f97dae729a70622cd1931f8 Mon Sep 17 00:00:00 2001 From: Logan Snow Date: Mon, 30 Mar 2026 14:32:20 -0400 Subject: [PATCH 5/7] Fix lazy imports: verify module exists before deferring Use importlib.util.find_spec() to check that a module actually exists before creating a lazy stub. Without this, conditional imports like `try: import brotli except ImportError: ...` would silently succeed, returning a lazy stub, and only fail later when an attribute is accessed -- far from the try/except guard. Co-Authored-By: Claude Opus 4.6 (1M context) --- Lib/_lazy_imports.py | 16 +++++++++++++++- Lib/test/test_lazy_imports.py | 6 ++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Lib/_lazy_imports.py b/Lib/_lazy_imports.py index cbe3f14fbae1df..c02db17153b29b 100644 --- a/Lib/_lazy_imports.py +++ b/Lib/_lazy_imports.py @@ -17,6 +17,7 @@ """ import builtins +import importlib.util import sys import types @@ -181,7 +182,20 @@ def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): if name in _EXCLUDED: return _original_import(name, globals, locals, fromlist, level) - # Simple ``import foo`` -- create a lazy stub. + # Simple ``import foo`` -- verify the module exists before deferring. + # Without this check, conditional imports like + # try: import brotli + # except ImportError: ... + # would silently succeed (returning a lazy stub) and only fail later + # when an attribute is accessed, far from the try/except guard. + try: + spec = importlib.util.find_spec(name) + except (ModuleNotFoundError, ValueError): + spec = None + if spec is None: + # Module does not exist -- let the original import raise. + return _original_import(name, globals, locals, fromlist, level) + lazy = LazyModule(name) sys.modules[name] = lazy return lazy diff --git a/Lib/test/test_lazy_imports.py b/Lib/test/test_lazy_imports.py index 75480446ee436b..b4305a0dfc5213 100644 --- a/Lib/test/test_lazy_imports.py +++ b/Lib/test/test_lazy_imports.py @@ -90,12 +90,10 @@ def test_lazy_module_dir(self): self.assertIn("dedent", d) def test_import_error_propagates(self): - """Importing a nonexistent module raises ImportError on access.""" + """Importing a nonexistent module raises ImportError immediately.""" sys.modules.pop("nonexistent_module_xyz_123", None) - import nonexistent_module_xyz_123 as mod - self.assertIsInstance(mod, self._lazy_imports.LazyModule) with self.assertRaises(ImportError): - mod.some_attr + import nonexistent_module_xyz_123 def test_dotted_import_is_eager(self): """Dotted imports like ``import a.b`` are always eager.""" From 77091150c7ecb382770e83e7778f31a631ec3415 Mon Sep 17 00:00:00 2001 From: Logan Snow Date: Mon, 30 Mar 2026 16:08:12 -0400 Subject: [PATCH 6/7] Fix double-load bug: resolve lazy parents before entering bootstrap When `from sqlalchemy.engine import Engine` was called and `sqlalchemy` was a LazyModule, the bootstrap's _find_and_load_unlocked would access parent.__path__, triggering _resolve() mid-flight. The resolution would transitively load sqlalchemy.engine as a side effect, then the original _find_and_load_unlocked would resume and load it AGAIN from scratch, creating a new module object that lacked the 'base' attribute. Fix: resolve all lazy ancestors in the parent chain *before* calling _original_import, so the bootstrap never encounters a LazyModule when traversing __path__. Also use __getattribute__ instead of __getattr__ to properly proxy __dict__ and other built-in module attributes. Co-Authored-By: Claude Opus 4.6 (1M context) --- Lib/_lazy_imports.py | 55 +++++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/Lib/_lazy_imports.py b/Lib/_lazy_imports.py index c02db17153b29b..163f8de3aab4e3 100644 --- a/Lib/_lazy_imports.py +++ b/Lib/_lazy_imports.py @@ -123,10 +123,23 @@ def __getattribute__(self, attr): if attr in LazyModule._INTERNAL_ATTRS: return object.__getattribute__(self, attr) real = object.__getattribute__(self, "_lazy_real") - if real is not None: + if real is None: + real = object.__getattribute__(self, "_resolve")() + try: + return getattr(real, attr) + except AttributeError: + # The attribute may be a submodule that hasn't been imported + # yet (e.g. ``import sqlalchemy`` then ``sqlalchemy.engine``). + # Try importing ``.`` before giving up. + name = object.__getattribute__(self, "_lazy_name") + submodule_name = f"{name}.{attr}" + try: + _original_import(submodule_name) + except ImportError: + raise AttributeError( + f"module {name!r} has no attribute {attr!r}" + ) from None return getattr(real, attr) - # Not yet resolved -- resolve now. - return getattr(object.__getattribute__(self, "_resolve")(), attr) def __repr__(self): real = object.__getattribute__(self, "_lazy_real") @@ -146,6 +159,23 @@ def __dir__(self): # Import hook # ------------------------------------------------------------------ +def _resolve_parent_chain(name): + """Resolve any LazyModule stubs in the parent chain of *name*. + + For a dotted name like ``a.b.c``, check ``a`` and ``a.b`` in + ``sys.modules`` and resolve them if they are lazy stubs. This must + be done *before* calling into the bootstrap so that + ``_find_and_load_unlocked`` never encounters a LazyModule when it + reads ``parent.__path__``. + """ + parts = name.split(".") + for i in range(len(parts) - 1): + ancestor = ".".join(parts[: i + 1]) + mod = sys.modules.get(ancestor) + if isinstance(mod, LazyModule): + mod._resolve() + + def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): """Replacement for builtins.__import__ that defers simple imports.""" # Relative imports -- always eager. @@ -154,23 +184,20 @@ def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): # ``from X import Y`` -- eager, because IMPORT_FROM needs real attrs. if fromlist: - # If the module is currently a lazy stub, resolve it first so the - # real import machinery sees the real module (or nothing). - mod = sys.modules.get(name) - if isinstance(mod, LazyModule): - mod._resolve() + # Resolve any lazy stubs in the parent chain *before* entering the + # bootstrap's _find_and_load. If we let _find_and_load discover a + # LazyModule parent via sys.modules, it will trigger _resolve() + # *inside* _find_and_load_unlocked — and the resolution may + # transitively load the very module we're trying to import, causing + # a double-load that creates a fresh (incomplete) module object. + _resolve_parent_chain(name) return _original_import(name, globals, locals, fromlist, level) # Dotted imports (import a.b.c) -- always eager. These must actually # load submodules as a side effect; deferring them breaks code that # relies on ``a.b`` being populated after ``import a.b.c``. if "." in name: - # Resolve any lazy stub for the top-level package first so the - # real import machinery can traverse its __path__. - top = name.partition(".")[0] - mod = sys.modules.get(top) - if isinstance(mod, LazyModule): - mod._resolve() + _resolve_parent_chain(name) return _original_import(name, globals, locals, fromlist, level) # Already in sys.modules -- return as-is (may be lazy or real). From 404e0fd364803144d2d9c1b1b0308c2b9afce4cf Mon Sep 17 00:00:00 2001 From: Logan Snow Date: Fri, 3 Apr 2026 11:43:43 -0400 Subject: [PATCH 7/7] Add lazy from-import support with _LazyFromImport proxy Extends lazy imports to also defer `from X import Y` statements. When the env var is set, `from foo import bar` returns a transparent proxy that resolves on first use (call, attribute access, operators, class inheritance via __mro_entries__, etc). Key design decisions: - Only top-level from-imports are deferred; transitive imports during module loading (relative imports, resolve chains) stay eager via _import_depth tracking - __mro_entries__ delegates to the real object's implementation (e.g. Generic) so metaclass machinery works correctly - Comprehensive dunder methods on the proxy for transparent operation - Core stdlib modules excluded to avoid bootstrap issues Co-Authored-By: Claude Opus 4.6 (1M context) --- Lib/_lazy_imports.py | 518 ++++++++++++++++++++++++++++------ Lib/test/test_lazy_imports.py | 185 ++++++++---- 2 files changed, 558 insertions(+), 145 deletions(-) diff --git a/Lib/_lazy_imports.py b/Lib/_lazy_imports.py index 163f8de3aab4e3..9e2fc81a7b3a7a 100644 --- a/Lib/_lazy_imports.py +++ b/Lib/_lazy_imports.py @@ -1,13 +1,11 @@ """Lazy import machinery for CPython. -When enabled via the PY_FORCE_LAZY_IMPORTS environment variable, top-level -``import foo`` statements are deferred: the module is not actually loaded -until one of its attributes is first accessed. ``from foo import bar`` -statements remain eager because the IMPORT_FROM opcode needs to resolve -the attribute immediately. +When enabled via the PY_FORCE_LAZY_IMPORTS environment variable, both +``import foo`` and ``from foo import bar`` statements are deferred. The +actual module is not loaded until the imported name is first used. -This is a simplified implementation inspired by PEP 690 and the approach -described by Hudson River Trading. +This is inspired by PEP 690 and the approach described by Hudson River +Trading. Usage:: @@ -25,45 +23,71 @@ _original_import = None +# Sentinel for unresolved proxy state. +_UNRESOLVED = object() + +# Depth counter: when > 0 we are inside a resolve / module load, so all +# from-imports should be eager to avoid interfering with module init code. +_import_depth = 0 + # Modules that must never be imported lazily. These are either critical # to the import system itself or have load-time side effects that many # other modules depend on. _EXCLUDED = frozenset({ + # Import machinery "importlib", "importlib._bootstrap", "importlib._bootstrap_external", "importlib.abc", "importlib.machinery", "importlib.util", "_frozen_importlib", "_frozen_importlib_external", "_imp", "_thread", "_weakref", "_io", "sys", "builtins", "marshal", "posix", "nt", "winreg", + # OS / path "errno", "os", "os.path", "posixpath", "ntpath", + # Signal "_signal", "signal", - "site", "_sitebuiltins", - "_lazy_imports", - # encodings are needed during startup + # Site + "site", "_sitebuiltins", "_lazy_imports", + # Encodings "encodings", "codecs", "_codecs", - # warnings infrastructure + # Warnings "warnings", "_warnings", - # abc machinery + # ABC "abc", "_abc", - # threading used by import lock + # Threading (import lock) "threading", "_threading", + # Core types / collections used pervasively by stdlib + "types", + "enum", "_enum", + "functools", "_functools", + "collections", "collections.abc", "_collections", "_collections_abc", + "operator", "_operator", + "itertools", "copyreg", + "re", "_sre", "sre_compile", "sre_parse", "sre_constants", + "struct", "_struct", + # IO / string + "io", "_io", "string", + # Misc core + "contextlib", "dataclasses", "inspect", "linecache", "tokenize", + "traceback", "weakref", "_weakrefset", }) +# Prefixes of excluded modules — computed once. +_EXCLUDED_PREFIXES = tuple(e + "." for e in _EXCLUDED) -class LazyModule(types.ModuleType): - """A module subclass that defers the actual import until first attribute access. - When an attribute is looked up on a ``LazyModule``, the real module is - loaded via the original ``__import__``, the ``LazyModule`` entry in - ``sys.modules`` is replaced with the real module, and the attribute - value is returned transparently. - """ +def _is_excluded(name): + if name in _EXCLUDED: + return True + return name.startswith(_EXCLUDED_PREFIXES) - # Use slots to keep lazy bookkeeping separate from normal module attrs. - __slots__ = ( - "_lazy_name", - "_lazy_real", - "_lazy_loading", - ) + +# ====================================================================== +# LazyModule — defers ``import foo`` +# ====================================================================== + +class LazyModule(types.ModuleType): + """A module subclass that defers the actual import until first attribute access.""" + + __slots__ = ("_lazy_name", "_lazy_real", "_lazy_loading") def __init__(self, name): super().__init__(name) @@ -71,55 +95,36 @@ def __init__(self, name): object.__setattr__(self, "_lazy_real", None) object.__setattr__(self, "_lazy_loading", False) - # ------------------------------------------------------------------ - # Resolution - # ------------------------------------------------------------------ - def _resolve(self): """Load the real module, replace ourselves in sys.modules, return it.""" + global _import_depth real = object.__getattribute__(self, "_lazy_real") if real is not None: return real if object.__getattribute__(self, "_lazy_loading"): - # Re-entrant call while we are already loading -- fall through - # to avoid infinite recursion. return self object.__setattr__(self, "_lazy_loading", True) name = object.__getattribute__(self, "_lazy_name") + _import_depth += 1 try: - # Remove ourselves from sys.modules so the real import machinery - # does not short-circuit back to us. sys.modules.pop(name, None) _original_import(name) real = sys.modules[name] object.__setattr__(self, "_lazy_real", real) return real except Exception: - # If the import fails, put ourselves back so the next attempt - # can retry. sys.modules.setdefault(name, self) raise finally: + _import_depth -= 1 object.__setattr__(self, "_lazy_loading", False) - # ------------------------------------------------------------------ - # Proxy dunder methods - # ------------------------------------------------------------------ - - # We need __getattribute__ (not just __getattr__) because attributes - # like __dict__, __name__, __spec__ etc. exist on the LazyModule - # itself (set by ModuleType.__init__). Using only __getattr__ would - # let those stale values leak out instead of delegating to the real - # module -- this breaks e.g. enum._convert_(source=lazy_mod) which - # iterates source.__dict__. - _INTERNAL_ATTRS = frozenset({ "_lazy_name", "_lazy_real", "_lazy_loading", "_resolve", "__class__", }) def __getattribute__(self, attr): - # Fast path: internal bookkeeping attrs must not trigger resolution. if attr in LazyModule._INTERNAL_ATTRS: return object.__getattribute__(self, attr) real = object.__getattribute__(self, "_lazy_real") @@ -128,9 +133,6 @@ def __getattribute__(self, attr): try: return getattr(real, attr) except AttributeError: - # The attribute may be a submodule that hasn't been imported - # yet (e.g. ``import sqlalchemy`` then ``sqlalchemy.engine``). - # Try importing ``.`` before giving up. name = object.__getattribute__(self, "_lazy_name") submodule_name = f"{name}.{attr}" try: @@ -155,19 +157,343 @@ def __dir__(self): return dir(object.__getattribute__(self, "_resolve")()) -# ------------------------------------------------------------------ -# Import hook -# ------------------------------------------------------------------ +# ====================================================================== +# _LazyFromImport — defers ``from foo import bar`` +# ====================================================================== -def _resolve_parent_chain(name): - """Resolve any LazyModule stubs in the parent chain of *name*. +class _LazyFromImport: + """Transparent proxy for a name obtained via ``from X import Y``. + + On first use (call, attribute access, operator, etc.) the proxy + resolves: imports the real module, extracts the attribute, replaces + itself in the caller's globals, caches the result, and delegates. + """ + + __slots__ = ( + "_lfi_module_name", "_lfi_attr_name", "_lfi_caller_globals", + "_lfi_resolved", "_lfi_resolving", + ) + + _LFI_INTERNAL = frozenset({ + "_lfi_module_name", "_lfi_attr_name", "_lfi_caller_globals", + "_lfi_resolved", "_lfi_resolving", "_resolve", + "__mro_entries__", # must be on proxy, not delegated + }) + + def __init__(self, module_name, attr_name, caller_globals): + object.__setattr__(self, "_lfi_module_name", module_name) + object.__setattr__(self, "_lfi_attr_name", attr_name) + object.__setattr__(self, "_lfi_caller_globals", caller_globals) + object.__setattr__(self, "_lfi_resolved", _UNRESOLVED) + object.__setattr__(self, "_lfi_resolving", False) + + def _resolve(self): + global _import_depth + resolved = object.__getattribute__(self, "_lfi_resolved") + if resolved is not _UNRESOLVED: + return resolved + if object.__getattribute__(self, "_lfi_resolving"): + return self + object.__setattr__(self, "_lfi_resolving", True) + mod_name = object.__getattribute__(self, "_lfi_module_name") + attr_name = object.__getattribute__(self, "_lfi_attr_name") + caller_globals = object.__getattribute__(self, "_lfi_caller_globals") + _import_depth += 1 + try: + _resolve_parent_chain(mod_name) + _original_import(mod_name) + mod = sys.modules[mod_name] + try: + real = getattr(mod, attr_name) + except AttributeError: + # Might be a submodule: from email import mime + sub = f"{mod_name}.{attr_name}" + try: + _original_import(sub) + real = sys.modules[sub] + except (ImportError, KeyError): + raise ImportError( + f"cannot import name {attr_name!r} from {mod_name!r}" + ) from None + object.__setattr__(self, "_lfi_resolved", real) + # Replace ourselves in the caller's namespace (handles aliases). + if caller_globals is not None: + for key, val in caller_globals.items(): + if val is self: + caller_globals[key] = real + break + return real + finally: + _import_depth -= 1 + object.__setattr__(self, "_lfi_resolving", False) + + # -- attribute access ----------------------------------------------- + + def __getattribute__(self, name): + if name in _LazyFromImport._LFI_INTERNAL: + return object.__getattribute__(self, name) + return getattr(object.__getattribute__(self, "_resolve")(), name) + + def __setattr__(self, name, value): + if name in _LazyFromImport._LFI_INTERNAL: + object.__setattr__(self, name, value) + return + setattr(object.__getattribute__(self, "_resolve")(), name, value) + + def __delattr__(self, name): + delattr(object.__getattribute__(self, "_resolve")(), name) + + # -- string --------------------------------------------------------- + + def __repr__(self): + resolved = object.__getattribute__(self, "_lfi_resolved") + if resolved is not _UNRESOLVED: + return repr(resolved) + mod = object.__getattribute__(self, "_lfi_module_name") + attr = object.__getattribute__(self, "_lfi_attr_name") + return f"" + + def __str__(self): + return str(object.__getattribute__(self, "_resolve")()) + + def __format__(self, spec): + return format(object.__getattribute__(self, "_resolve")(), spec) + + def __bytes__(self): + return bytes(object.__getattribute__(self, "_resolve")()) + + # -- class / type --------------------------------------------------- + + @property + def __class__(self): + resolved = object.__getattribute__(self, "_lfi_resolved") + if resolved is not _UNRESOLVED: + return type(resolved) + return _LazyFromImport + + @__class__.setter + def __class__(self, value): + pass # swallow; some code introspects __class__ + + def __mro_entries__(self, bases): + # PEP 560: called by type.__new__ to resolve non-type bases. + # This lets the proxy be used in ``class Foo(LazyProxy): ...`` + real = object.__getattribute__(self, "_resolve")() + # If the resolved object has its own __mro_entries__ (e.g. Generic), + # delegate to it so the MRO is constructed correctly. + own_mro = getattr(type(real), "__mro_entries__", None) + if own_mro is not None: + return own_mro(real, bases) + return (real,) + + def __instancecheck__(self, instance): + return isinstance(instance, object.__getattribute__(self, "_resolve")()) + + def __subclasscheck__(self, subclass): + return issubclass(subclass, object.__getattribute__(self, "_resolve")()) + + # -- callable ------------------------------------------------------- + + def __call__(self, *args, **kwargs): + return object.__getattribute__(self, "_resolve")()(*args, **kwargs) + + # -- comparison / hashing ------------------------------------------- + + def __eq__(self, other): + return object.__getattribute__(self, "_resolve")() == other + + def __ne__(self, other): + return object.__getattribute__(self, "_resolve")() != other + + def __lt__(self, other): + return object.__getattribute__(self, "_resolve")() < other + + def __le__(self, other): + return object.__getattribute__(self, "_resolve")() <= other + + def __gt__(self, other): + return object.__getattribute__(self, "_resolve")() > other + + def __ge__(self, other): + return object.__getattribute__(self, "_resolve")() >= other + + def __hash__(self): + return hash(object.__getattribute__(self, "_resolve")()) + + def __bool__(self): + return bool(object.__getattribute__(self, "_resolve")()) + + # -- container ------------------------------------------------------ + + def __len__(self): + return len(object.__getattribute__(self, "_resolve")()) + + def __getitem__(self, key): + return object.__getattribute__(self, "_resolve")()[key] + + def __setitem__(self, key, value): + object.__getattribute__(self, "_resolve")()[key] = value + + def __delitem__(self, key): + del object.__getattribute__(self, "_resolve")()[key] + + def __contains__(self, item): + return item in object.__getattribute__(self, "_resolve")() - For a dotted name like ``a.b.c``, check ``a`` and ``a.b`` in - ``sys.modules`` and resolve them if they are lazy stubs. This must - be done *before* calling into the bootstrap so that - ``_find_and_load_unlocked`` never encounters a LazyModule when it - reads ``parent.__path__``. + def __iter__(self): + return iter(object.__getattribute__(self, "_resolve")()) + + def __next__(self): + return next(object.__getattribute__(self, "_resolve")()) + + def __reversed__(self): + return reversed(object.__getattribute__(self, "_resolve")()) + + # -- arithmetic ----------------------------------------------------- + + def __add__(self, other): + return object.__getattribute__(self, "_resolve")() + other + + def __radd__(self, other): + return other + object.__getattribute__(self, "_resolve")() + + def __sub__(self, other): + return object.__getattribute__(self, "_resolve")() - other + + def __rsub__(self, other): + return other - object.__getattribute__(self, "_resolve")() + + def __mul__(self, other): + return object.__getattribute__(self, "_resolve")() * other + + def __rmul__(self, other): + return other * object.__getattribute__(self, "_resolve")() + + def __truediv__(self, other): + return object.__getattribute__(self, "_resolve")() / other + + def __rtruediv__(self, other): + return other / object.__getattribute__(self, "_resolve")() + + def __floordiv__(self, other): + return object.__getattribute__(self, "_resolve")() // other + + def __mod__(self, other): + return object.__getattribute__(self, "_resolve")() % other + + def __pow__(self, other, mod=None): + return pow(object.__getattribute__(self, "_resolve")(), other, mod) + + def __neg__(self): + return -object.__getattribute__(self, "_resolve")() + + def __pos__(self): + return +object.__getattribute__(self, "_resolve")() + + def __abs__(self): + return abs(object.__getattribute__(self, "_resolve")()) + + def __invert__(self): + return ~object.__getattribute__(self, "_resolve")() + + def __int__(self): + return int(object.__getattribute__(self, "_resolve")()) + + def __float__(self): + return float(object.__getattribute__(self, "_resolve")()) + + def __index__(self): + return object.__getattribute__(self, "_resolve")().__index__() + + # -- bitwise / union (int | str style) ------------------------------ + + def __or__(self, other): + return object.__getattribute__(self, "_resolve")() | other + + def __ror__(self, other): + return other | object.__getattribute__(self, "_resolve")() + + def __and__(self, other): + return object.__getattribute__(self, "_resolve")() & other + + def __rand__(self, other): + return other & object.__getattribute__(self, "_resolve")() + + def __xor__(self, other): + return object.__getattribute__(self, "_resolve")() ^ other + + # -- context manager ------------------------------------------------ + + def __enter__(self): + return object.__getattribute__(self, "_resolve")().__enter__() + + def __exit__(self, *args): + return object.__getattribute__(self, "_resolve")().__exit__(*args) + + # -- async ---------------------------------------------------------- + + def __await__(self): + return object.__getattribute__(self, "_resolve")().__await__() + + def __aiter__(self): + return object.__getattribute__(self, "_resolve")().__aiter__() + + def __anext__(self): + return object.__getattribute__(self, "_resolve")().__anext__() + + def __aenter__(self): + return object.__getattribute__(self, "_resolve")().__aenter__() + + def __aexit__(self, *args): + return object.__getattribute__(self, "_resolve")().__aexit__(*args) + + # -- misc ----------------------------------------------------------- + + def __dir__(self): + return dir(object.__getattribute__(self, "_resolve")()) + + def __fspath__(self): + import os + return os.fspath(object.__getattribute__(self, "_resolve")()) + + +# ====================================================================== +# _DeferredFromModule — returned by __import__ for lazy from-imports +# ====================================================================== + +class _DeferredFromModule: + """Lightweight stand-in returned by ``__import__`` for ``from X import Y``. + + ``IMPORT_FROM`` calls ``getattr(module, name)`` which hits our + ``__getattr__``, returning a ``_LazyFromImport`` proxy for each name. """ + + __slots__ = ("_dfm_name", "_dfm_globals") + + def __init__(self, module_name, caller_globals): + object.__setattr__(self, "_dfm_name", module_name) + object.__setattr__(self, "_dfm_globals", caller_globals) + + def __getattr__(self, name): + mod_name = object.__getattribute__(self, "_dfm_name") + caller_globals = object.__getattribute__(self, "_dfm_globals") + return _LazyFromImport(mod_name, name, caller_globals) + + @property + def __name__(self): + # ceval.c import_from() reads __name__ for the sys.modules fallback. + return object.__getattribute__(self, "_dfm_name") + + def __repr__(self): + return f"" + + +# ====================================================================== +# Import hook +# ====================================================================== + +def _resolve_parent_chain(name): + """Resolve any LazyModule stubs in the parent chain of *name*.""" parts = name.split(".") for i in range(len(parts) - 1): ancestor = ".".join(parts[: i + 1]) @@ -177,25 +503,53 @@ def _resolve_parent_chain(name): def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): - """Replacement for builtins.__import__ that defers simple imports.""" - # Relative imports -- always eager. + """Replacement for builtins.__import__ that defers imports.""" + # Relative imports -- always eager (with depth tracking so that + # transitive from-imports during the load are also eager). if level > 0: - return _original_import(name, globals, locals, fromlist, level) + global _import_depth + _import_depth += 1 + try: + return _original_import(name, globals, locals, fromlist, level) + finally: + _import_depth -= 1 - # ``from X import Y`` -- eager, because IMPORT_FROM needs real attrs. + # ``from X import Y`` handling. if fromlist: - # Resolve any lazy stubs in the parent chain *before* entering the - # bootstrap's _find_and_load. If we let _find_and_load discover a - # LazyModule parent via sys.modules, it will trigger _resolve() - # *inside* _find_and_load_unlocked — and the resolution may - # transitively load the very module we're trying to import, causing - # a double-load that creates a fresh (incomplete) module object. + # Always-eager cases: + # - inside a resolve / eager import (transitive imports) + # - from __future__ import ... + # - from X import * (needs __all__ / __dict__) + # - excluded modules + # - no caller globals (can't do self-replacement) + # - module already loaded (cheap to do eagerly) + if (_import_depth > 0 + or name == "__future__" + or fromlist == ("*",) + or _is_excluded(name) + or globals is None): + _resolve_parent_chain(name) + return _original_import(name, globals, locals, fromlist, level) + + # If the module is already fully loaded, just do it eagerly — + # it's only a dict lookup + getattr at that point. + mod = sys.modules.get(name) + if mod is not None and not isinstance(mod, LazyModule): + return _original_import(name, globals, locals, fromlist, level) + + # Verify the module exists before deferring. _resolve_parent_chain(name) - return _original_import(name, globals, locals, fromlist, level) + try: + spec = importlib.util.find_spec(name) + except (ModuleNotFoundError, ValueError): + spec = None + if spec is None: + return _original_import(name, globals, locals, fromlist, level) + + # Defer! + return _DeferredFromModule(name, globals) - # Dotted imports (import a.b.c) -- always eager. These must actually - # load submodules as a side effect; deferring them breaks code that - # relies on ``a.b`` being populated after ``import a.b.c``. + # Dotted imports (import a.b.c) -- always eager. if "." in name: _resolve_parent_chain(name) return _original_import(name, globals, locals, fromlist, level) @@ -206,21 +560,15 @@ def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): return mod # Excluded modules -- always eager. - if name in _EXCLUDED: + if _is_excluded(name): return _original_import(name, globals, locals, fromlist, level) # Simple ``import foo`` -- verify the module exists before deferring. - # Without this check, conditional imports like - # try: import brotli - # except ImportError: ... - # would silently succeed (returning a lazy stub) and only fail later - # when an attribute is accessed, far from the try/except guard. try: spec = importlib.util.find_spec(name) except (ModuleNotFoundError, ValueError): spec = None if spec is None: - # Module does not exist -- let the original import raise. return _original_import(name, globals, locals, fromlist, level) lazy = LazyModule(name) @@ -228,15 +576,15 @@ def _lazy_import_hook(name, globals=None, locals=None, fromlist=(), level=0): return lazy -# ------------------------------------------------------------------ +# ====================================================================== # Public API -# ------------------------------------------------------------------ +# ====================================================================== def install(): """Replace ``builtins.__import__`` with the lazy import hook.""" global _original_import if _original_import is not None: - return # already installed + return _original_import = builtins.__import__ builtins.__import__ = _lazy_import_hook diff --git a/Lib/test/test_lazy_imports.py b/Lib/test/test_lazy_imports.py index b4305a0dfc5213..3367d3fdb7cdff 100644 --- a/Lib/test/test_lazy_imports.py +++ b/Lib/test/test_lazy_imports.py @@ -13,19 +13,16 @@ class TestLazyModule(unittest.TestCase): def setUp(self): import _lazy_imports self._lazy_imports = _lazy_imports - # Save original state. self._orig_modules = sys.modules.copy() self._lazy_imports.install() def tearDown(self): self._lazy_imports.uninstall() - # Restore sys.modules to avoid leaking state between tests. sys.modules.clear() sys.modules.update(self._orig_modules) def test_import_creates_lazy_module(self): """A plain ``import`` of an unloaded module creates a LazyModule stub.""" - # Pick a stdlib module unlikely to be loaded yet in a fresh test. sys.modules.pop("decimal", None) import decimal self.assertIsInstance(decimal, self._lazy_imports.LazyModule) @@ -35,25 +32,110 @@ def test_lazy_module_resolves_on_attr_access(self): sys.modules.pop("textwrap", None) import textwrap self.assertIsInstance(textwrap, self._lazy_imports.LazyModule) - # This attribute access should trigger the real import. result = textwrap.dedent(" hello") self.assertEqual(result, "hello") - # After resolution, sys.modules should hold the real module. self.assertNotIsInstance(sys.modules["textwrap"], self._lazy_imports.LazyModule) - def test_from_import_is_eager(self): - """``from X import Y`` must work and be resolved eagerly.""" + def test_from_import_creates_lazy_proxy(self): + """``from X import Y`` creates a _LazyFromImport proxy.""" sys.modules.pop("base64", None) from base64 import b64encode - self.assertTrue(callable(b64encode)) - # The module itself should have been loaded (not lazy). - self.assertNotIsInstance(sys.modules.get("base64"), + self.assertIsInstance(b64encode, self._lazy_imports._LazyFromImport) + + def test_from_import_proxy_resolves_on_call(self): + """Calling a lazy proxy triggers resolution.""" + sys.modules.pop("base64", None) + from base64 import b64encode + result = b64encode(b"hello") + self.assertEqual(result, b"aGVsbG8=") + # After resolution, base64 should be loaded. + self.assertIn("base64", sys.modules) + self.assertNotIsInstance(sys.modules["base64"], self._lazy_imports.LazyModule) + def test_from_import_proxy_resolves_on_attr(self): + """Accessing an attribute on a lazy proxy triggers resolution.""" + sys.modules.pop("decimal", None) + from decimal import Decimal + name = Decimal.__name__ + self.assertEqual(name, "Decimal") + + def test_from_import_proxy_replaces_in_globals(self): + """After resolution, the caller's globals contain the real object.""" + sys.modules.pop("textwrap", None) + g = {} + exec("from textwrap import dedent", g) + proxy = g["dedent"] + self.assertIsInstance(proxy, self._lazy_imports._LazyFromImport) + # Trigger resolution. + exec("result = dedent(' hi')", g) + self.assertEqual(g["result"], "hi") + # The proxy should have been replaced. + self.assertNotIsInstance(g["dedent"], + self._lazy_imports._LazyFromImport) + + def test_from_import_multiple_names(self): + """``from X import Y, Z`` creates independent proxies.""" + sys.modules.pop("base64", None) + from base64 import b64encode, b64decode + self.assertIsInstance(b64encode, self._lazy_imports._LazyFromImport) + self.assertIsInstance(b64decode, self._lazy_imports._LazyFromImport) + # Resolve one — the other should still be lazy (until used). + b64encode(b"test") + # b64decode may or may not have been replaced depending on + # whether it's in the same globals dict; just test it works. + self.assertEqual(b64decode(b"dGVzdA=="), b"test") + + def test_from_import_star_is_eager(self): + """``from X import *`` is always eager.""" + sys.modules.pop("base64", None) + g = {} + exec("from base64 import *", g) + self.assertTrue(callable(g.get("b64encode"))) + self.assertNotIsInstance(g["b64encode"], + self._lazy_imports._LazyFromImport) + + def test_from_import_excluded_is_eager(self): + """Excluded modules are imported eagerly.""" + sys.modules.pop("os", None) + from os import path + self.assertNotIsInstance(path, self._lazy_imports._LazyFromImport) + + def test_from_import_nonexistent_module(self): + """``from nonexistent import X`` raises ImportError immediately.""" + with self.assertRaises(ImportError): + from nonexistent_module_xyz_123 import something + + def test_from_import_class_as_constructor(self): + """A lazily imported class can be used as a constructor.""" + sys.modules.pop("collections", None) + from collections import OrderedDict + d = OrderedDict(a=1, b=2) + self.assertEqual(list(d.keys()), ["a", "b"]) + + def test_from_import_constant_arithmetic(self): + """A lazily imported constant works in arithmetic.""" + sys.modules.pop("math", None) + from math import pi + self.assertAlmostEqual(pi, 3.14159265, places=5) + self.assertAlmostEqual(pi + 1, 4.14159265, places=5) + + def test_from_import_proxy_repr(self): + """Proxy repr is informative before and after resolution.""" + sys.modules.pop("textwrap", None) + from textwrap import dedent + r = repr(dedent) + self.assertIn("lazy", r) + self.assertIn("textwrap", r) + # Resolve + dedent(" hi") + r = repr(dedent) + self.assertIn("function", r.lower()) + def test_already_loaded_module_is_returned_directly(self): """If a module is already in sys.modules, return it as-is.""" - import json # force eager + import json json_mod = sys.modules["json"] import json as json2 self.assertIs(json2, json_mod) @@ -77,8 +159,6 @@ def test_lazy_module_repr(self): sys.modules.pop("decimal", None) import decimal self.assertIn("lazy module", repr(decimal)) - self.assertIn("decimal", repr(decimal)) - # Resolve it. decimal.Decimal self.assertNotIn("lazy module", repr(decimal)) @@ -101,18 +181,13 @@ def test_dotted_import_is_eager(self): sys.modules.pop("email.mime", None) sys.modules.pop("email.mime.text", None) import email.mime.text - # The submodule must actually be loaded (not lazy). self.assertNotIsInstance(sys.modules.get("email"), self._lazy_imports.LazyModule) - self.assertNotIsInstance(sys.modules.get("email.mime.text"), - self._lazy_imports.LazyModule) - # And the submodule must be accessible. self.assertTrue(hasattr(email.mime.text, "MIMEText")) class TestEnvironmentVariable(unittest.TestCase): - """Integration tests that verify the PY_FORCE_LAZY_IMPORTS env var works - end-to-end via subprocess.""" + """Integration tests via subprocess.""" def _run_python(self, code, env_val=None): env = os.environ.copy() @@ -122,42 +197,28 @@ def _run_python(self, code, env_val=None): env.pop("PY_FORCE_LAZY_IMPORTS", None) return subprocess.run( [sys.executable, "-c", code], - capture_output=True, text=True, env=env, - timeout=30, + capture_output=True, text=True, env=env, timeout=30, ) def test_env_var_enables_lazy_imports(self): - """With PY_FORCE_LAZY_IMPORTS=true, imports are lazy.""" - code = ( - "import sys\n" - "import _lazy_imports\n" - "print(_lazy_imports.is_enabled())\n" - ) + code = "import _lazy_imports; print(_lazy_imports.is_enabled())" result = self._run_python(code, env_val="true") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout.strip(), "True") def test_env_var_disabled_by_default(self): - """Without PY_FORCE_LAZY_IMPORTS, imports are not lazy.""" - code = ( - "import _lazy_imports\n" - "print(_lazy_imports.is_enabled())\n" - ) + code = "import _lazy_imports; print(_lazy_imports.is_enabled())" result = self._run_python(code, env_val=None) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout.strip(), "False") def test_lazy_import_defers_loading(self): - """Prove that a module is NOT in sys.modules until accessed.""" code = ( "import sys\n" - # decimal should not be loaded at startup with lazy imports - "# Now import decimal lazily\n" "import decimal\n" "loaded_before = 'decimal' in sys.modules and " "not hasattr(sys.modules['decimal'], '_lazy_name')\n" "print('before:', loaded_before)\n" - "# Now trigger the actual load\n" "decimal.Decimal('1.0')\n" "loaded_after = 'decimal' in sys.modules and " "not hasattr(sys.modules['decimal'], '_lazy_name')\n" @@ -166,34 +227,36 @@ def test_lazy_import_defers_loading(self): result = self._run_python(code, env_val="1") self.assertEqual(result.returncode, 0, result.stderr) lines = result.stdout.strip().splitlines() - self.assertEqual(lines[0], "before: False", - "Module should not be really loaded before access") - self.assertEqual(lines[1], "after: True", - "Module should be really loaded after access") + self.assertEqual(lines[0], "before: False") + self.assertEqual(lines[1], "after: True") - def test_from_import_works_with_lazy(self): - """``from X import Y`` works correctly under lazy imports.""" + def test_from_import_lazy_end_to_end(self): + """``from X import Y`` is lazy in a subprocess.""" code = ( - "from base64 import b64encode\n" - "print(b64encode(b'hello').decode())\n" + "import sys\n" + "from textwrap import dedent\n" + "# Module should not be loaded yet\n" + "mod = sys.modules.get('textwrap')\n" + "print('before:', mod is None or hasattr(mod, '_lazy_name'))\n" + "# Now use it\n" + "print('result:', dedent(' hi'))\n" + "mod = sys.modules.get('textwrap')\n" + "print('after:', mod is not None and not hasattr(mod, '_lazy_name'))\n" ) result = self._run_python(code, env_val="true") self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "aGVsbG8=") + lines = result.stdout.strip().splitlines() + self.assertEqual(lines[0], "before: True") + self.assertEqual(lines[1], "result: hi") + self.assertEqual(lines[2], "after: True") - def test_lazy_import_functional(self): - """A lazily imported module works correctly end-to-end.""" - code = ( - "import json\n" - "result = json.dumps({'key': 'value'})\n" - "print(result)\n" - ) + def test_from_import_works_with_lazy(self): + code = "from base64 import b64encode; print(b64encode(b'hello').decode())" result = self._run_python(code, env_val="true") self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), '{"key": "value"}') + self.assertEqual(result.stdout.strip(), "aGVsbG8=") def test_dotted_import_works_with_lazy(self): - """Dotted imports work correctly under lazy imports.""" code = ( "import email.mime.text\n" "msg = email.mime.text.MIMEText('hello')\n" @@ -203,22 +266,24 @@ def test_dotted_import_works_with_lazy(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout.strip(), "MIMEText") + def test_lazy_import_functional(self): + code = "import json; print(json.dumps({'key': 'value'}))" + result = self._run_python(code, env_val="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), '{"key": "value"}') + def test_env_var_accepts_multiple_values(self): - """PY_FORCE_LAZY_IMPORTS accepts '1', 'true', 'yes' (case insensitive).""" code = "import _lazy_imports; print(_lazy_imports.is_enabled())" for val in ("1", "true", "True", "TRUE", "yes", "Yes"): with self.subTest(val=val): result = self._run_python(code, env_val=val) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "True", - f"PY_FORCE_LAZY_IMPORTS={val} should enable lazy imports") - + self.assertEqual(result.stdout.strip(), "True") for val in ("0", "false", "no", ""): with self.subTest(val=val): result = self._run_python(code, env_val=val) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.strip(), "False", - f"PY_FORCE_LAZY_IMPORTS={val} should NOT enable lazy imports") + self.assertEqual(result.stdout.strip(), "False") if __name__ == "__main__":