pdoc.extract

  1"""
  2This module handles the interaction with Python's module system,
  3that is it loads the correct module based on whatever the user specified,
  4and provides the rest of pdoc with some additional module metadata.
  5"""
  6
  7from __future__ import annotations
  8
  9from collections.abc import Iterable
 10from collections.abc import Iterator
 11from collections.abc import Sequence
 12from contextlib import contextmanager
 13import importlib.util
 14import io
 15import linecache
 16import os
 17from pathlib import Path
 18import pkgutil
 19import platform
 20import re
 21import shutil
 22import subprocess
 23import sys
 24import traceback
 25import types
 26from unittest.mock import patch
 27import warnings
 28
 29import pdoc.doc_ast
 30import pdoc.docstrings
 31
 32
 33def walk_specs(specs: Sequence[Path | str]) -> list[str]:
 34    """
 35    This function processes a list of module specifications and returns a collection of module names, including all
 36    submodules, that should be processed by pdoc.
 37
 38    A module specification can either be the name of an installed module, or the path to a specific file or package.
 39    For example, the following strings are valid module specifications:
 40
 41     - `typing`
 42     - `collections.abc`
 43     - `./test/testdata/demo_long.py`
 44     - `./test/testdata/demopackage`
 45
 46    *This function has side effects:* See `parse_spec`.
 47    """
 48    all_modules: dict[str, None] = {}
 49    for spec in specs:
 50        if isinstance(spec, str) and spec.startswith("!"):
 51            ignore_pattern = re.compile(spec[1:])
 52            all_modules = {
 53                k: v for k, v in all_modules.items() if not ignore_pattern.match(k)
 54            }
 55            continue
 56
 57        modname = parse_spec(spec)
 58
 59        try:
 60            with mock_some_common_side_effects():
 61                modspec = importlib.util.find_spec(modname)
 62                if modspec is None:
 63                    raise ModuleNotFoundError(modname)
 64        except AnyException:
 65            warnings.warn(
 66                f"Cannot find spec for {modname} (from {spec}):\n{traceback.format_exc()}",
 67                stacklevel=2,
 68            )
 69        else:
 70            mod_info = pkgutil.ModuleInfo(
 71                None,  # type: ignore
 72                name=modname,
 73                ispkg=bool(modspec.submodule_search_locations),
 74            )
 75            for m in walk_packages2([mod_info]):
 76                if m.name in all_modules:
 77                    warnings.warn(
 78                        f"The module specification {spec!r} adds a module named {m.name}, but a module with this name "
 79                        f"has already been added. You may have accidentally repeated a module spec, or you are trying "
 80                        f"to document two modules with the same filename from two different directories, which does "
 81                        f"not work. Only one documentation page will be generated."
 82                    )
 83                all_modules[m.name] = None
 84
 85    if not all_modules:
 86        raise ValueError(
 87            f"No modules found matching spec: {', '.join(str(x) for x in specs)}"
 88        )
 89
 90    return list(all_modules)
 91
 92
 93def parse_spec(spec: Path | str) -> str:
 94    """
 95    This functions parses a user's module specification into a module identifier that can be imported.
 96    If both a local file/directory and an importable module with the same name exist, a warning will be printed.
 97
 98    *This function has side effects:* `sys.path` will be amended if the specification is a path.
 99    If this side effect is undesired, pass a module name instead.
100    """
101    pspec = Path(spec)
102    if isinstance(spec, str) and (os.sep in spec or (os.altsep and os.altsep in spec)):
103        # We have a path separator, so it's definitely a filepath.
104        spec = pspec
105
106    if isinstance(spec, str) and (pspec.is_file() or (pspec / "__init__.py").is_file()):
107        # We have a local file with this name, but is there also a module with the same name?
108        try:
109            with mock_some_common_side_effects():
110                modspec = importlib.util.find_spec(spec)
111                if modspec is None:
112                    raise ModuleNotFoundError
113        except AnyException:
114            # Module does not exist, use local file.
115            spec = pspec
116        else:
117            # Module does exist. We now check if the local file/directory is the same (e.g. after pip install -e),
118            # and emit a warning if that's not the case.
119            origin = (
120                Path(modspec.origin).absolute() if modspec.origin else Path("unknown")
121            )
122            local_dir = Path(spec).absolute()
123            if local_dir not in (origin, origin.parent):
124                warnings.warn(
125                    f"{spec!r} may refer to either the installed Python module or the local file/directory with the "
126                    f"same name. pdoc will document the installed module, prepend './' to force documentation of the "
127                    f"local file/directory.\n"
128                    f" - Module location: {origin}\n"
129                    f" - Local file/directory: {local_dir}",
130                    RuntimeWarning,
131                )
132
133    if isinstance(spec, Path):
134        if spec.name == "__init__.py":
135            spec = spec.parent
136        if (spec.parent / "__init__.py").exists():
137            return parse_spec(spec.resolve().parent) + f".{spec.stem}"
138        parent_dir = str(spec.parent)
139        sys.path = [parent_dir] + [x for x in sys.path if x != parent_dir]
140        if spec.stem in sys.modules and sys.modules[spec.stem].__file__:
141            local_dir = spec.resolve()
142            file = sys.modules[spec.stem].__file__
143            assert file is not None  # make mypy happy
144            origin = Path(file).resolve()
145            if local_dir not in (origin, origin.parent, origin.with_suffix("")):
146                warnings.warn(
147                    f"pdoc cannot load {spec.stem!r} because a module with the same name is already imported in pdoc's "
148                    f"Python process. pdoc will document the loaded module from {origin} instead.",
149                    RuntimeWarning,
150                )
151        return spec.stem
152    else:
153        return spec
154
155
156def _noop(*args, **kwargs):
157    pass
158
159
160class _PdocDefusedPopen(subprocess.Popen):
161    """A small wrapper around subprocess.Popen that converts most executions into no-ops."""
162
163    if platform.system() == "Windows":  # pragma: no cover
164        _noop_exe = "echo.exe"
165    else:  # pragma: no cover
166        _noop_exe = "echo"
167
168    def __init__(self, *args, **kwargs):  # pragma: no cover
169        command_allowed = (
170            args
171            and args[0]
172            and args[0][0]
173            in (
174                # these invocations may all come from https://github.com/python/cpython/blob/main/Lib/ctypes/util.py,
175                # which we want to keep working.
176                "/sbin/ldconfig",
177                "ld",
178                shutil.which("gcc") or shutil.which("cc"),
179                shutil.which("objdump"),
180                # https://github.com/mitmproxy/pdoc/issues/430: GitPython invokes git commands, which is also fine.
181                "git",
182            )
183        )
184        if not command_allowed and os.environ.get("PDOC_ALLOW_EXEC", "") == "":
185            # sys.stderr is patched, so we need to unpatch it for printing a warning.
186            with patch("sys.stderr", new=sys.__stderr__):
187                warnings.warn(
188                    f"Suppressed execution of {args[0]!r} during import. "
189                    f"Set PDOC_ALLOW_EXEC=1 as an environment variable to allow subprocess execution.",
190                    stacklevel=2,
191                )
192            kwargs["executable"] = self._noop_exe
193        super().__init__(*args, **kwargs)
194
195
196@contextmanager
197def mock_some_common_side_effects():
198    """
199    This context manager is applied when importing modules. It mocks some common side effects that may happen upon
200    module import. For example, `import antigravity` normally causes a web browser to open, which we want to suppress.
201
202    Note that this function must not be used for security purposes, it's easily bypassable.
203    """
204    with patch("subprocess.Popen", new=_PdocDefusedPopen), patch(
205        "os.startfile", new=_noop, create=True
206    ), patch("sys.stdout", new=io.StringIO()), patch(
207        "sys.stderr", new=io.StringIO()
208    ), patch("sys.stdin", new=io.StringIO()):
209        yield
210
211
212@mock_some_common_side_effects()
213def load_module(module: str) -> types.ModuleType:
214    """Try to import a module. If import fails, a RuntimeError is raised.
215
216    Returns the imported module."""
217    try:
218        return importlib.import_module(module)
219    except AnyException as e:
220        raise RuntimeError(f"Error importing {module}") from e
221
222
223AnyException = (SystemExit, GeneratorExit, Exception)
224"""BaseException, but excluding KeyboardInterrupt.
225
226Modules may raise SystemExit on import (which we want to catch),
227but we don't want to catch a user's KeyboardInterrupt.
228"""
229
230
231def iter_modules2(module: types.ModuleType) -> dict[str, pkgutil.ModuleInfo]:
232    """
233    Returns all direct child modules of a given module.
234    This function is similar to `pkgutil.iter_modules`, but
235
236      1. Respects a package's `__all__` attribute if specified.
237         If `__all__` is defined, submodules not listed in `__all__` are excluded.
238      2. It will try to detect submodules that are not findable with iter_modules,
239         but are present in the module object.
240    """
241    mod_all = getattr(module, "__all__", None)
242
243    submodules = {}
244
245    for submodule in pkgutil.iter_modules(
246        getattr(module, "__path__", []), f"{module.__name__}."
247    ):
248        name = submodule.name.rpartition(".")[2]
249        if mod_all is None or name in mod_all:
250            submodules[name] = submodule
251
252    # 2023-12: PyO3 and pybind11 submodules are not detected by pkgutil
253    # This is a hacky workaround to register them.
254    members = dir(module) if mod_all is None else mod_all
255    for name in members:
256        if name in submodules or name == "__main__" or not isinstance(name, str):
257            continue
258        member = getattr(module, name, None)
259        is_wild_child_module = (
260            isinstance(member, types.ModuleType)
261            # the name is either just "bar", but can also be "foo.bar",
262            # see https://github.com/PyO3/pyo3/issues/759#issuecomment-1811992321
263            and (
264                member.__name__ == f"{module.__name__}.{name}"
265                or (
266                    member.__name__ == name
267                    and sys.modules.get(member.__name__, None) is not member
268                )
269            )
270        )
271        if is_wild_child_module:
272            # fixup the module name so that the rest of pdoc does not break
273            assert member
274            member.__name__ = f"{module.__name__}.{name}"
275            sys.modules[f"{module.__name__}.{name}"] = member
276            submodules[name] = pkgutil.ModuleInfo(
277                None,  # type: ignore
278                name=f"{module.__name__}.{name}",
279                ispkg=True,
280            )
281
282    submodules.pop("__main__", None)  # https://github.com/mitmproxy/pdoc/issues/438
283
284    return submodules
285
286
287def walk_packages2(
288    modules: Iterable[pkgutil.ModuleInfo],
289) -> Iterator[pkgutil.ModuleInfo]:
290    """
291    For a given list of modules, recursively yield their names and all their submodules' names.
292
293    This function is similar to `pkgutil.walk_packages`, but based on `iter_modules2`.
294    """
295    # the original walk_packages implementation has a recursion check for path, but that does not seem to be needed?
296    for mod in modules:
297        yield mod
298
299        if mod.ispkg:
300            try:
301                module = load_module(mod.name)
302            except RuntimeError:
303                warnings.warn(f"Error loading {mod.name}:\n{traceback.format_exc()}")
304                continue
305
306            submodules = iter_modules2(module)
307            yield from walk_packages2(submodules.values())
308
309
310def module_mtime(modulename: str) -> float | None:
311    """Returns the time the specified module file was last modified, or `None` if this cannot be determined.
312    The primary use of this is live-reloading modules on modification."""
313    try:
314        with mock_some_common_side_effects():
315            spec = importlib.util.find_spec(modulename)
316    except AnyException:
317        pass
318    else:
319        if spec is not None and spec.origin is not None:
320            return Path(spec.origin).stat().st_mtime
321    return None
322
323
324def invalidate_caches(module_name: str) -> None:
325    """
326    Invalidate module cache to allow live-reloading of modules.
327    """
328    # Getting this right is tricky – reloading modules causes a bunch of surprising side effects.
329    # Our current best effort is to call `importlib.reload` on all modules that start with module_name.
330    # We also exclude our own dependencies, which cause fun errors otherwise.
331    if module_name not in sys.modules:
332        return
333    if any(
334        module_name.startswith(f"{x}.") or x == module_name
335        for x in ("jinja2", "markupsafe", "markdown2", "pygments")
336    ):
337        return
338
339    # a more extreme alternative:
340    # filename = sys.modules[module_name].__file__
341    # if (
342    #    filename.startswith(sysconfig.get_path("platstdlib"))
343    #    or filename.startswith(sysconfig.get_path("stdlib"))
344    # ):
345    #     return
346
347    importlib.invalidate_caches()
348    linecache.clearcache()
349    pdoc.doc.Module.from_name.cache_clear()
350    pdoc.doc_ast._get_source.cache_clear()
351    pdoc.docstrings.convert.cache_clear()
352
353    prefix = f"{module_name}."
354    mods = sorted(
355        mod for mod in sys.modules if module_name == mod or mod.startswith(prefix)
356    )
357    for modname in mods:
358        if modname == "pdoc.render":
359            # pdoc.render is stateful after configure(), so we don't want to reload it.
360            continue
361        try:
362            if not isinstance(sys.modules[modname], types.ModuleType):
363                continue  # some funky stuff going on - one example is typing.io, which is a class.
364            with mock_some_common_side_effects():
365                importlib.reload(sys.modules[modname])
366        except AnyException:
367            warnings.warn(
368                f"Error reloading {modname}:\n{traceback.format_exc()}",
369                stacklevel=2,
370            )