pdoc.extract

  1"""
  2This module handles the interaction with Python's module system,
  3that is it loads the correct module based on whatever the user specified,
  4and provides the rest of pdoc with some additional module metadata.
  5"""
  6
  7from __future__ import annotations
  8
  9from collections.abc import Iterable
 10from collections.abc import Iterator
 11from collections.abc import Sequence
 12from contextlib import contextmanager
 13import importlib.util
 14import io
 15import linecache
 16import os
 17from pathlib import Path
 18import pkgutil
 19import platform
 20import re
 21import shutil
 22import subprocess
 23import sys
 24import traceback
 25import types
 26from unittest.mock import patch
 27import warnings
 28
 29import pdoc.doc_ast
 30import pdoc.docstrings
 31
 32
 33def walk_specs(specs: Sequence[Path | str]) -> list[str]:
 34    """
 35    This function processes a list of module specifications and returns a collection of module names, including all
 36    submodules, that should be processed by pdoc.
 37
 38    A module specification can either be the name of an installed module, or the path to a specific file or package.
 39    For example, the following strings are valid module specifications:
 40
 41     - `typing`
 42     - `collections.abc`
 43     - `./test/testdata/demo_long.py`
 44     - `./test/testdata/demopackage`
 45
 46    *This function has side effects:* See `parse_spec`.
 47    """
 48    all_modules: dict[str, None] = {}
 49    for spec in specs:
 50        if isinstance(spec, str) and spec.startswith("!"):
 51            ignore_pattern = re.compile(spec[1:])
 52            all_modules = {
 53                k: v for k, v in all_modules.items() if not ignore_pattern.match(k)
 54            }
 55            continue
 56
 57        modname = parse_spec(spec)
 58
 59        try:
 60            with mock_some_common_side_effects():
 61                modspec = importlib.util.find_spec(modname)
 62                if modspec is None:
 63                    raise ModuleNotFoundError(modname)
 64        except pdoc.docstrings.AnyException:
 65            warnings.warn(
 66                f"Cannot find spec for {modname} (from {spec}):\n{traceback.format_exc()}",
 67                stacklevel=2,
 68            )
 69        else:
 70            mod_info = pkgutil.ModuleInfo(
 71                None,  # type: ignore
 72                name=modname,
 73                ispkg=bool(modspec.submodule_search_locations),
 74            )
 75            for m in walk_packages2([mod_info]):
 76                if m.name in all_modules:
 77                    warnings.warn(
 78                        f"The module specification {spec!r} adds a module named {m.name}, but a module with this name "
 79                        f"has already been added. You may have accidentally repeated a module spec, or you are trying "
 80                        f"to document two modules with the same filename from two different directories, which does "
 81                        f"not work. Only one documentation page will be generated."
 82                    )
 83                all_modules[m.name] = None
 84
 85    if not all_modules:
 86        raise ValueError(
 87            f"No modules found matching spec: {', '.join(str(x) for x in specs)}"
 88        )
 89
 90    return list(all_modules)
 91
 92
 93def parse_spec(spec: Path | str) -> str:
 94    """
 95    This functions parses a user's module specification into a module identifier that can be imported.
 96    If both a local file/directory and an importable module with the same name exist, a warning will be printed.
 97
 98    *This function has side effects:* `sys.path` will be amended if the specification is a path.
 99    If this side effect is undesired, pass a module name instead.
100    """
101    pspec = Path(spec)
102    if isinstance(spec, str) and (os.sep in spec or (os.altsep and os.altsep in spec)):
103        # We have a path separator, so it's definitely a filepath.
104        spec = pspec
105
106    if isinstance(spec, str) and (pspec.is_file() or (pspec / "__init__.py").is_file()):
107        # We have a local file with this name, but is there also a module with the same name?
108        try:
109            with mock_some_common_side_effects():
110                modspec = importlib.util.find_spec(spec)
111                if modspec is None:
112                    raise ModuleNotFoundError
113        except pdoc.docstrings.AnyException:
114            # Module does not exist, use local file.
115            spec = pspec.resolve()
116        else:
117            # Module does exist. We now check if the local file/directory is the same (e.g. after pip install -e),
118            # and emit a warning if that's not the case.
119            origin = (
120                Path(modspec.origin).absolute() if modspec.origin else Path("unknown")
121            )
122            local_dir = Path(spec).absolute()
123            if local_dir not in (origin, origin.parent):
124                warnings.warn(
125                    f"{spec!r} may refer to either the installed Python module or the local file/directory with the "
126                    f"same name. pdoc will document the installed module, prepend './' to force documentation of the "
127                    f"local file/directory.\n"
128                    f" - Module location: {origin}\n"
129                    f" - Local file/directory: {local_dir}",
130                    RuntimeWarning,
131                )
132
133    if isinstance(spec, Path):
134        if spec.name == "__init__.py":
135            spec = spec.parent
136        if (spec.parent / "__init__.py").exists():
137            return parse_spec(spec.resolve().parent) + f".{spec.stem}"
138        parent_dir = str(spec.parent)
139        sys.path = [parent_dir] + [x for x in sys.path if x != parent_dir]
140        if spec.stem in sys.modules and sys.modules[spec.stem].__file__:
141            local_dir = spec.resolve()
142            file = sys.modules[spec.stem].__file__
143            assert file is not None  # make mypy happy
144            origin = Path(file).resolve()
145            if local_dir not in (origin, origin.parent, origin.with_suffix("")):
146                warnings.warn(
147                    f"pdoc cannot load {spec.stem!r} because a module with the same name is already imported in pdoc's "
148                    f"Python process. pdoc will document the loaded module from {origin} instead.",
149                    RuntimeWarning,
150                )
151        return spec.stem
152    else:
153        return spec
154
155
156def _noop(*args, **kwargs):
157    pass
158
159
160class _PdocDefusedPopen(subprocess.Popen):
161    """A small wrapper around subprocess.Popen that converts most executions into no-ops."""
162
163    if platform.system() == "Windows":  # pragma: no cover
164        _noop_exe = "echo.exe"
165    else:  # pragma: no cover
166        _noop_exe = "echo"
167
168    def __init__(self, *args, **kwargs):  # pragma: no cover
169        command_allowed = (
170            args
171            and args[0]
172            and args[0][0]
173            in (
174                # these invocations may all come from https://github.com/python/cpython/blob/main/Lib/ctypes/util.py,
175                # which we want to keep working.
176                "/sbin/ldconfig",
177                "ld",
178                shutil.which("gcc") or shutil.which("cc"),
179                shutil.which("objdump"),
180                # https://github.com/mitmproxy/pdoc/issues/430: GitPython invokes git commands, which is also fine.
181                "git",
182            )
183        )
184        if not command_allowed and os.environ.get("PDOC_ALLOW_EXEC", "") == "":
185            # sys.stderr is patched, so we need to unpatch it for printing a warning.
186            with patch("sys.stderr", new=sys.__stderr__):
187                warnings.warn(
188                    f"Suppressed execution of {args[0]!r} during import. "
189                    f"Set PDOC_ALLOW_EXEC=1 as an environment variable to allow subprocess execution.",
190                    stacklevel=2,
191                )
192            kwargs["executable"] = self._noop_exe
193        super().__init__(*args, **kwargs)
194
195
196@contextmanager
197def mock_some_common_side_effects():
198    """
199    This context manager is applied when importing modules. It mocks some common side effects that may happen upon
200    module import. For example, `import antigravity` normally causes a web browser to open, which we want to suppress.
201
202    Note that this function must not be used for security purposes, it's easily bypassable.
203    """
204    with (
205        patch("subprocess.Popen", new=_PdocDefusedPopen),
206        patch("os.startfile", new=_noop, create=True),
207        patch("sys.stdout", new=io.TextIOWrapper(io.BytesIO())),
208        patch("sys.stderr", new=io.TextIOWrapper(io.BytesIO())),
209        patch("sys.stdin", new=io.TextIOWrapper(io.BytesIO())),
210    ):
211        yield
212
213
214@mock_some_common_side_effects()
215def load_module(module: str) -> types.ModuleType:
216    """Try to import a module. If import fails, a RuntimeError is raised.
217
218    Returns the imported module."""
219    try:
220        return importlib.import_module(module)
221    except pdoc.docstrings.AnyException as e:
222        raise RuntimeError(f"Error importing {module}") from e
223
224
225def iter_modules2(module: types.ModuleType) -> dict[str, pkgutil.ModuleInfo]:
226    """
227    Returns all direct child modules of a given module.
228    This function is similar to `pkgutil.iter_modules`, but
229
230      1. Respects a package's `__all__` attribute if specified.
231         If `__all__` is defined, submodules not listed in `__all__` are excluded.
232      2. It will try to detect submodules that are not findable with iter_modules,
233         but are present in the module object.
234    """
235    mod_all = getattr(module, "__all__", None)
236
237    submodules = {}
238
239    for submodule in pkgutil.iter_modules(
240        getattr(module, "__path__", []), f"{module.__name__}."
241    ):
242        name = submodule.name.rpartition(".")[2]
243        if mod_all is None or name in mod_all:
244            submodules[name] = submodule
245
246    # 2023-12: PyO3 and pybind11 submodules are not detected by pkgutil
247    # This is a hacky workaround to register them.
248    members = dir(module) if mod_all is None else mod_all
249    for name in members:
250        if name in submodules or name == "__main__" or not isinstance(name, str):
251            continue
252        member = getattr(module, name, None)
253        is_wild_child_module = (
254            isinstance(member, types.ModuleType)
255            # the name is either just "bar", but can also be "foo.bar",
256            # see https://github.com/PyO3/pyo3/issues/759#issuecomment-1811992321
257            and (
258                member.__name__ == f"{module.__name__}.{name}"
259                or (
260                    member.__name__ == name
261                    and sys.modules.get(member.__name__, None) is not member
262                )
263            )
264        )
265        if is_wild_child_module:
266            # fixup the module name so that the rest of pdoc does not break
267            assert member
268            member.__name__ = f"{module.__name__}.{name}"
269            sys.modules[f"{module.__name__}.{name}"] = member
270            submodules[name] = pkgutil.ModuleInfo(
271                None,  # type: ignore
272                name=f"{module.__name__}.{name}",
273                ispkg=True,
274            )
275
276    submodules.pop("__main__", None)  # https://github.com/mitmproxy/pdoc/issues/438
277
278    return submodules
279
280
281def walk_packages2(
282    modules: Iterable[pkgutil.ModuleInfo],
283) -> Iterator[pkgutil.ModuleInfo]:
284    """
285    For a given list of modules, recursively yield their names and all their submodules' names.
286
287    This function is similar to `pkgutil.walk_packages`, but based on `iter_modules2`.
288    """
289    # the original walk_packages implementation has a recursion check for path, but that does not seem to be needed?
290    for mod in modules:
291        yield mod
292
293        if mod.ispkg:
294            try:
295                module = load_module(mod.name)
296            except RuntimeError:
297                warnings.warn(f"Error loading {mod.name}:\n{traceback.format_exc()}")
298                continue
299
300            submodules = iter_modules2(module)
301            yield from walk_packages2(submodules.values())
302
303
304def module_mtime(modulename: str) -> float | None:
305    """Returns the time the specified module file was last modified, or `None` if this cannot be determined.
306    The primary use of this is live-reloading modules on modification."""
307    try:
308        with mock_some_common_side_effects():
309            spec = importlib.util.find_spec(modulename)
310    except pdoc.docstrings.AnyException:
311        pass
312    else:
313        if spec is not None and spec.origin is not None:
314            return Path(spec.origin).stat().st_mtime
315    return None
316
317
318def invalidate_caches(module_name: str) -> None:
319    """
320    Invalidate module cache to allow live-reloading of modules.
321    """
322    # Getting this right is tricky – reloading modules causes a bunch of surprising side effects.
323    # Our current best effort is to call `importlib.reload` on all modules that start with module_name.
324    # We also exclude our own dependencies, which cause fun errors otherwise.
325    if module_name not in sys.modules:
326        return
327    if any(
328        module_name.startswith(f"{x}.") or x == module_name
329        for x in ("jinja2", "markupsafe", "markdown2", "pygments")
330    ):
331        return
332
333    # a more extreme alternative:
334    # filename = sys.modules[module_name].__file__
335    # if (
336    #    filename.startswith(sysconfig.get_path("platstdlib"))
337    #    or filename.startswith(sysconfig.get_path("stdlib"))
338    # ):
339    #     return
340
341    importlib.invalidate_caches()
342    linecache.clearcache()
343    pdoc.doc.Module.from_name.cache_clear()
344    pdoc.doc_ast._get_source.cache_clear()
345    pdoc.docstrings.convert.cache_clear()
346
347    prefix = f"{module_name}."
348    mods = sorted(
349        mod for mod in sys.modules if module_name == mod or mod.startswith(prefix)
350    )
351    for modname in mods:
352        if modname == "pdoc.render":
353            # pdoc.render is stateful after configure(), so we don't want to reload it.
354            continue
355        try:
356            if not isinstance(sys.modules[modname], types.ModuleType):
357                continue  # some funky stuff going on - one example is typing.io, which is a class.
358            with mock_some_common_side_effects():
359                importlib.reload(sys.modules[modname])
360        except pdoc.docstrings.AnyException:
361            warnings.warn(
362                f"Error reloading {modname}:\n{traceback.format_exc()}",
363                stacklevel=2,
364            )