pdoc.extract

  1"""
  2This module handles the interaction with Python's module system,
  3that is it loads the correct module based on whatever the user specified,
  4and provides the rest of pdoc with some additional module metadata.
  5"""
  6
  7from __future__ import annotations
  8
  9from collections.abc import Iterable
 10from collections.abc import Iterator
 11from collections.abc import Sequence
 12from contextlib import contextmanager
 13import importlib.util
 14import io
 15import linecache
 16import os
 17from pathlib import Path
 18import pkgutil
 19import platform
 20import re
 21import shutil
 22import subprocess
 23import sys
 24import traceback
 25import types
 26from unittest.mock import patch
 27import warnings
 28
 29import pdoc.doc_ast
 30import pdoc.docstrings
 31
 32
 33def walk_specs(specs: Sequence[Path | str]) -> list[str]:
 34    """
 35    This function processes a list of module specifications and returns a collection of module names, including all
 36    submodules, that should be processed by pdoc.
 37
 38    A module specification can either be the name of an installed module, or the path to a specific file or package.
 39    For example, the following strings are valid module specifications:
 40
 41     - `typing`
 42     - `collections.abc`
 43     - `./test/testdata/demo_long.py`
 44     - `./test/testdata/demopackage`
 45
 46    *This function has side effects:* See `parse_spec`.
 47    """
 48    all_modules: dict[str, None] = {}
 49    for spec in specs:
 50        if isinstance(spec, str) and spec.startswith("!"):
 51            ignore_pattern = re.compile(spec[1:])
 52            all_modules = {
 53                k: v for k, v in all_modules.items() if not ignore_pattern.match(k)
 54            }
 55            continue
 56
 57        modname = parse_spec(spec)
 58
 59        try:
 60            with mock_some_common_side_effects():
 61                modspec = importlib.util.find_spec(modname)
 62                if modspec is None:
 63                    raise ModuleNotFoundError(modname)
 64        except AnyException:
 65            warnings.warn(
 66                f"Cannot find spec for {modname} (from {spec}):\n{traceback.format_exc()}",
 67                stacklevel=2,
 68            )
 69        else:
 70            mod_info = pkgutil.ModuleInfo(
 71                None,  # type: ignore
 72                name=modname,
 73                ispkg=bool(modspec.submodule_search_locations),
 74            )
 75            for m in walk_packages2([mod_info]):
 76                if m.name in all_modules:
 77                    warnings.warn(
 78                        f"The module specification {spec!r} adds a module named {m.name}, but a module with this name "
 79                        f"has already been added. You may have accidentally repeated a module spec, or you are trying "
 80                        f"to document two modules with the same filename from two different directories, which does "
 81                        f"not work. Only one documentation page will be generated."
 82                    )
 83                all_modules[m.name] = None
 84
 85    if not all_modules:
 86        raise ValueError(
 87            f"No modules found matching spec: {', '.join(str(x) for x in specs)}"
 88        )
 89
 90    return list(all_modules)
 91
 92
 93def parse_spec(spec: Path | str) -> str:
 94    """
 95    This functions parses a user's module specification into a module identifier that can be imported.
 96    If both a local file/directory and an importable module with the same name exist, a warning will be printed.
 97
 98    *This function has side effects:* `sys.path` will be amended if the specification is a path.
 99    If this side effect is undesired, pass a module name instead.
100    """
101    pspec = Path(spec)
102    if isinstance(spec, str) and (os.sep in spec or (os.altsep and os.altsep in spec)):
103        # We have a path separator, so it's definitely a filepath.
104        spec = pspec
105
106    if isinstance(spec, str) and (pspec.is_file() or (pspec / "__init__.py").is_file()):
107        # We have a local file with this name, but is there also a module with the same name?
108        try:
109            with mock_some_common_side_effects():
110                modspec = importlib.util.find_spec(spec)
111                if modspec is None:
112                    raise ModuleNotFoundError
113        except AnyException:
114            # Module does not exist, use local file.
115            spec = pspec
116        else:
117            # Module does exist. We now check if the local file/directory is the same (e.g. after pip install -e),
118            # and emit a warning if that's not the case.
119            origin = (
120                Path(modspec.origin).absolute() if modspec.origin else Path("unknown")
121            )
122            local_dir = Path(spec).absolute()
123            if local_dir not in (origin, origin.parent):
124                warnings.warn(
125                    f"{spec!r} may refer to either the installed Python module or the local file/directory with the "
126                    f"same name. pdoc will document the installed module, prepend './' to force documentation of the "
127                    f"local file/directory.\n"
128                    f" - Module location: {origin}\n"
129                    f" - Local file/directory: {local_dir}",
130                    RuntimeWarning,
131                )
132
133    if isinstance(spec, Path):
134        if spec.name == "__init__.py":
135            spec = spec.parent
136        if (spec.parent / "__init__.py").exists():
137            return parse_spec(spec.resolve().parent) + f".{spec.stem}"
138        parent_dir = str(spec.parent)
139        sys.path = [parent_dir] + [x for x in sys.path if x != parent_dir]
140        if spec.stem in sys.modules and sys.modules[spec.stem].__file__:
141            local_dir = spec.resolve()
142            file = sys.modules[spec.stem].__file__
143            assert file is not None  # make mypy happy
144            origin = Path(file).resolve()
145            if local_dir not in (origin, origin.parent, origin.with_suffix("")):
146                warnings.warn(
147                    f"pdoc cannot load {spec.stem!r} because a module with the same name is already imported in pdoc's "
148                    f"Python process. pdoc will document the loaded module from {origin} instead.",
149                    RuntimeWarning,
150                )
151        return spec.stem
152    else:
153        return spec
154
155
156def _noop(*args, **kwargs):
157    pass
158
159
160class _PdocDefusedPopen(subprocess.Popen):
161    """A small wrapper around subprocess.Popen that converts most executions into no-ops."""
162
163    if platform.system() == "Windows":  # pragma: no cover
164        _noop_exe = "echo.exe"
165    else:  # pragma: no cover
166        _noop_exe = "echo"
167
168    def __init__(self, *args, **kwargs):  # pragma: no cover
169        command_allowed = (
170            args
171            and args[0]
172            and args[0][0]
173            in (
174                # these invocations may all come from https://github.com/python/cpython/blob/main/Lib/ctypes/util.py,
175                # which we want to keep working.
176                "/sbin/ldconfig",
177                "ld",
178                shutil.which("gcc") or shutil.which("cc"),
179                shutil.which("objdump"),
180                # https://github.com/mitmproxy/pdoc/issues/430: GitPython invokes git commands, which is also fine.
181                "git",
182            )
183        )
184        if not command_allowed and os.environ.get("PDOC_ALLOW_EXEC", "") == "":
185            # sys.stderr is patched, so we need to unpatch it for printing a warning.
186            with patch("sys.stderr", new=sys.__stderr__):
187                warnings.warn(
188                    f"Suppressed execution of {args[0]!r} during import. "
189                    f"Set PDOC_ALLOW_EXEC=1 as an environment variable to allow subprocess execution.",
190                    stacklevel=2,
191                )
192            kwargs["executable"] = self._noop_exe
193        super().__init__(*args, **kwargs)
194
195
196@contextmanager
197def mock_some_common_side_effects():
198    """
199    This context manager is applied when importing modules. It mocks some common side effects that may happen upon
200    module import. For example, `import antigravity` normally causes a web browser to open, which we want to suppress.
201
202    Note that this function must not be used for security purposes, it's easily bypassable.
203    """
204    with (
205        patch("subprocess.Popen", new=_PdocDefusedPopen),
206        patch("os.startfile", new=_noop, create=True),
207        patch("sys.stdout", new=io.TextIOWrapper(io.BytesIO())),
208        patch("sys.stderr", new=io.TextIOWrapper(io.BytesIO())),
209        patch("sys.stdin", new=io.TextIOWrapper(io.BytesIO())),
210    ):
211        yield
212
213
214@mock_some_common_side_effects()
215def load_module(module: str) -> types.ModuleType:
216    """Try to import a module. If import fails, a RuntimeError is raised.
217
218    Returns the imported module."""
219    try:
220        return importlib.import_module(module)
221    except AnyException as e:
222        raise RuntimeError(f"Error importing {module}") from e
223
224
225AnyException = (SystemExit, GeneratorExit, Exception)
226"""BaseException, but excluding KeyboardInterrupt.
227
228Modules may raise SystemExit on import (which we want to catch),
229but we don't want to catch a user's KeyboardInterrupt.
230"""
231
232
233def iter_modules2(module: types.ModuleType) -> dict[str, pkgutil.ModuleInfo]:
234    """
235    Returns all direct child modules of a given module.
236    This function is similar to `pkgutil.iter_modules`, but
237
238      1. Respects a package's `__all__` attribute if specified.
239         If `__all__` is defined, submodules not listed in `__all__` are excluded.
240      2. It will try to detect submodules that are not findable with iter_modules,
241         but are present in the module object.
242    """
243    mod_all = getattr(module, "__all__", None)
244
245    submodules = {}
246
247    for submodule in pkgutil.iter_modules(
248        getattr(module, "__path__", []), f"{module.__name__}."
249    ):
250        name = submodule.name.rpartition(".")[2]
251        if mod_all is None or name in mod_all:
252            submodules[name] = submodule
253
254    # 2023-12: PyO3 and pybind11 submodules are not detected by pkgutil
255    # This is a hacky workaround to register them.
256    members = dir(module) if mod_all is None else mod_all
257    for name in members:
258        if name in submodules or name == "__main__" or not isinstance(name, str):
259            continue
260        member = getattr(module, name, None)
261        is_wild_child_module = (
262            isinstance(member, types.ModuleType)
263            # the name is either just "bar", but can also be "foo.bar",
264            # see https://github.com/PyO3/pyo3/issues/759#issuecomment-1811992321
265            and (
266                member.__name__ == f"{module.__name__}.{name}"
267                or (
268                    member.__name__ == name
269                    and sys.modules.get(member.__name__, None) is not member
270                )
271            )
272        )
273        if is_wild_child_module:
274            # fixup the module name so that the rest of pdoc does not break
275            assert member
276            member.__name__ = f"{module.__name__}.{name}"
277            sys.modules[f"{module.__name__}.{name}"] = member
278            submodules[name] = pkgutil.ModuleInfo(
279                None,  # type: ignore
280                name=f"{module.__name__}.{name}",
281                ispkg=True,
282            )
283
284    submodules.pop("__main__", None)  # https://github.com/mitmproxy/pdoc/issues/438
285
286    return submodules
287
288
289def walk_packages2(
290    modules: Iterable[pkgutil.ModuleInfo],
291) -> Iterator[pkgutil.ModuleInfo]:
292    """
293    For a given list of modules, recursively yield their names and all their submodules' names.
294
295    This function is similar to `pkgutil.walk_packages`, but based on `iter_modules2`.
296    """
297    # the original walk_packages implementation has a recursion check for path, but that does not seem to be needed?
298    for mod in modules:
299        yield mod
300
301        if mod.ispkg:
302            try:
303                module = load_module(mod.name)
304            except RuntimeError:
305                warnings.warn(f"Error loading {mod.name}:\n{traceback.format_exc()}")
306                continue
307
308            submodules = iter_modules2(module)
309            yield from walk_packages2(submodules.values())
310
311
312def module_mtime(modulename: str) -> float | None:
313    """Returns the time the specified module file was last modified, or `None` if this cannot be determined.
314    The primary use of this is live-reloading modules on modification."""
315    try:
316        with mock_some_common_side_effects():
317            spec = importlib.util.find_spec(modulename)
318    except AnyException:
319        pass
320    else:
321        if spec is not None and spec.origin is not None:
322            return Path(spec.origin).stat().st_mtime
323    return None
324
325
326def invalidate_caches(module_name: str) -> None:
327    """
328    Invalidate module cache to allow live-reloading of modules.
329    """
330    # Getting this right is tricky – reloading modules causes a bunch of surprising side effects.
331    # Our current best effort is to call `importlib.reload` on all modules that start with module_name.
332    # We also exclude our own dependencies, which cause fun errors otherwise.
333    if module_name not in sys.modules:
334        return
335    if any(
336        module_name.startswith(f"{x}.") or x == module_name
337        for x in ("jinja2", "markupsafe", "markdown2", "pygments")
338    ):
339        return
340
341    # a more extreme alternative:
342    # filename = sys.modules[module_name].__file__
343    # if (
344    #    filename.startswith(sysconfig.get_path("platstdlib"))
345    #    or filename.startswith(sysconfig.get_path("stdlib"))
346    # ):
347    #     return
348
349    importlib.invalidate_caches()
350    linecache.clearcache()
351    pdoc.doc.Module.from_name.cache_clear()
352    pdoc.doc_ast._get_source.cache_clear()
353    pdoc.docstrings.convert.cache_clear()
354
355    prefix = f"{module_name}."
356    mods = sorted(
357        mod for mod in sys.modules if module_name == mod or mod.startswith(prefix)
358    )
359    for modname in mods:
360        if modname == "pdoc.render":
361            # pdoc.render is stateful after configure(), so we don't want to reload it.
362            continue
363        try:
364            if not isinstance(sys.modules[modname], types.ModuleType):
365                continue  # some funky stuff going on - one example is typing.io, which is a class.
366            with mock_some_common_side_effects():
367                importlib.reload(sys.modules[modname])
368        except AnyException:
369            warnings.warn(
370                f"Error reloading {modname}:\n{traceback.format_exc()}",
371                stacklevel=2,
372            )