pdoc.extract

  1"""
  2This module handles the interaction with Python's module system,
  3that is it loads the correct module based on whatever the user specified,
  4and provides the rest of pdoc with some additional module metadata.
  5"""
  6from __future__ import annotations
  7
  8import importlib
  9import importlib.util
 10import io
 11import linecache
 12import os
 13import pkgutil
 14import platform
 15import re
 16import shutil
 17import subprocess
 18import sys
 19import traceback
 20import types
 21import warnings
 22from collections.abc import Callable, Iterable, Iterator, Sequence
 23from contextlib import contextmanager
 24from pathlib import Path
 25from unittest.mock import patch
 26
 27import pdoc
 28import pdoc.doc_ast
 29import pdoc.docstrings
 30
 31
 32def walk_specs(specs: Sequence[Path | str]) -> list[str]:
 33    """
 34    This function processes a list of module specifications and returns a collection of module names, including all
 35    submodules, that should be processed by pdoc.
 36
 37    A module specification can either be the name of an installed module, or the path to a specific file or package.
 38    For example, the following strings are valid module specifications:
 39
 40     - `typing`
 41     - `collections.abc`
 42     - `./test/testdata/demo_long.py`
 43     - `./test/testdata/demopackage`
 44
 45    *This function has side effects:* See `parse_spec`.
 46    """
 47    all_modules: dict[str, None] = {}
 48    for spec in specs:
 49
 50        if isinstance(spec, str) and spec.startswith("!"):
 51            ignore_pattern = re.compile(spec[1:])
 52            all_modules = {
 53                k: v for k, v in all_modules.items() if not ignore_pattern.match(k)
 54            }
 55            continue
 56
 57        modname = parse_spec(spec)
 58
 59        try:
 60            with mock_some_common_side_effects():
 61                modspec = importlib.util.find_spec(modname)
 62                if modspec is None:
 63                    raise ModuleNotFoundError(modname)
 64        except AnyException:
 65            warnings.warn(
 66                f"Cannot find spec for {modname} (from {spec}):\n{traceback.format_exc()}",
 67                stacklevel=2,
 68            )
 69        else:
 70            mod_info = pkgutil.ModuleInfo(
 71                None,  # type: ignore
 72                name=modname,
 73                ispkg=bool(modspec.submodule_search_locations),
 74            )
 75            for m in walk_packages2([mod_info]):
 76                if m.name in all_modules:
 77                    warnings.warn(
 78                        f"The module specification {spec!r} adds a module named {m.name}, but a module with this name "
 79                        f"has already been added. You may have accidentally repeated a module spec, or you are trying "
 80                        f"to document two modules with the same filename from two different directories, which does "
 81                        f"not work. Only one documentation page will be generated."
 82                    )
 83                all_modules[m.name] = None
 84
 85    if not all_modules:
 86        raise ValueError(
 87            f"No modules found matching spec: {', '.join(str(x) for x in specs)}"
 88        )
 89
 90    return list(all_modules)
 91
 92
 93def parse_spec(spec: Path | str) -> str:
 94    """
 95    This functions parses a user's module specification into a module identifier that can be imported.
 96    If both a local file/directory and an importable module with the same name exist, a warning will be printed.
 97
 98    *This function has side effects:* `sys.path` will be amended if the specification is a path.
 99    If this side effect is undesired, pass a module name instead.
100    """
101    pspec = Path(spec)
102    if isinstance(spec, str) and (os.sep in spec or (os.altsep and os.altsep in spec)):
103        # We have a path separator, so it's definitely a filepath.
104        spec = pspec
105
106    if isinstance(spec, str) and (pspec.is_file() or (pspec / "__init__.py").is_file()):
107        # We have a local file with this name, but is there also a module with the same name?
108        try:
109            with mock_some_common_side_effects():
110                modspec = importlib.util.find_spec(spec)
111                if modspec is None:
112                    raise ModuleNotFoundError
113        except AnyException:
114            # Module does not exist, use local file.
115            spec = pspec
116        else:
117            # Module does exist. We now check if the local file/directory is the same (e.g. after pip install -e),
118            # and emit a warning if that's not the case.
119            origin = (
120                Path(modspec.origin).absolute() if modspec.origin else Path("unknown")
121            )
122            local_dir = Path(spec).absolute()
123            if local_dir not in (origin, origin.parent):
124                warnings.warn(
125                    f"{spec!r} may refer to either the installed Python module or the local file/directory with the "
126                    f"same name. pdoc will document the installed module, prepend './' to force documentation of the "
127                    f"local file/directory.\n"
128                    f" - Module location: {origin}\n"
129                    f" - Local file/directory: {local_dir}",
130                    RuntimeWarning,
131                )
132
133    if isinstance(spec, Path):
134        if (spec.parent / "__init__.py").exists():
135            return parse_spec(spec.resolve().parent) + f".{spec.stem}"
136        parent_dir = str(spec.parent)
137        sys.path = [parent_dir] + [x for x in sys.path if x != parent_dir]
138        if spec.stem in sys.modules and sys.modules[spec.stem].__file__:
139            local_dir = spec.resolve()
140            file = sys.modules[spec.stem].__file__
141            assert file is not None  # make mypy happy
142            origin = Path(file).resolve()
143            if local_dir not in (origin, origin.parent, origin.with_suffix("")):
144                warnings.warn(
145                    f"pdoc cannot load {spec.stem!r} because a module with the same name is already imported in pdoc's "
146                    f"Python process. pdoc will document the loaded module from {origin} instead.",
147                    RuntimeWarning,
148                )
149        return spec.stem
150    else:
151        return spec
152
153
154def _noop(*args, **kwargs):
155    pass
156
157
158class PdocDefusedPopen(subprocess.Popen):
159    if platform.system() == "Windows":  # pragma: no cover
160        _noop_exe = "echo.exe"
161    else:  # pragma: no cover
162        _noop_exe = "echo"
163
164    def __init__(self, *args, **kwargs):  # pragma: no cover
165        command_allowed = (
166            args
167            and args[0]
168            and args[0][0]
169            in (
170                # these invocations may all come from https://github.com/python/cpython/blob/main/Lib/ctypes/util.py,
171                # which we want to keep working.
172                "/sbin/ldconfig",
173                "ld",
174                shutil.which("gcc") or shutil.which("cc"),
175                shutil.which("objdump"),
176            )
177        )
178        if not command_allowed:
179            kwargs["executable"] = self._noop_exe
180        super().__init__(*args, **kwargs)
181
182
183@contextmanager
184def mock_some_common_side_effects():
185    """
186    This context manager is applied when importing modules. It mocks some common side effects that may happen upon
187    module import. For example, `import antigravity` normally causes a web browser to open, which we want to suppress.
188
189    Note that this function must not be used for security purposes, it's easily bypassable.
190    """
191    with patch("subprocess.Popen", new=PdocDefusedPopen), patch(
192        "os.startfile", new=_noop, create=True
193    ), patch("sys.stdout", new=io.StringIO()), patch(
194        "sys.stderr", new=io.StringIO()
195    ), patch(
196        "sys.stdin", new=io.StringIO()
197    ):
198        yield
199
200
201@mock_some_common_side_effects()
202def load_module(module: str) -> types.ModuleType:
203    """Try to import a module. If import fails, a RuntimeError is raised.
204
205    Returns the imported module."""
206    try:
207        return importlib.import_module(module)
208    except AnyException as e:
209        raise RuntimeError(f"Error importing {module}") from e
210
211
212AnyException = (SystemExit, GeneratorExit, Exception)
213"""BaseException, but excluding KeyboardInterrupt.
214
215Modules may raise SystemExit on import (which we want to catch),
216but we don't want to catch a user's KeyboardInterrupt.
217"""
218
219
220def _all_submodules(modulename: str) -> bool:
221    return True
222
223
224def walk_packages2(
225    modules: Iterable[pkgutil.ModuleInfo],
226    module_filter: Callable[[str], bool] = _all_submodules,
227) -> Iterator[pkgutil.ModuleInfo]:
228    """
229    For a given list of modules, recursively yield their names and all their submodules' names.
230
231    This function is similar to `pkgutil.walk_packages`, but respects a package's `__all__` attribute if specified.
232    If `__all__` is defined, submodules not listed in `__all__` are excluded.
233    """
234
235    # noinspection PyDefaultArgument
236    def seen(p, m={}):  # pragma: no cover
237        if p in m:
238            return True
239        m[p] = True
240
241    for mod in modules:
242        # is __all__ defined and the module not in __all__?
243        if not module_filter(mod.name.rpartition(".")[2]):
244            continue
245
246        yield mod
247
248        if mod.ispkg:
249            try:
250                module = load_module(mod.name)
251            except RuntimeError:
252                warnings.warn(f"Error loading {mod.name}:\n{traceback.format_exc()}")
253                continue
254
255            mod_all = getattr(module, "__all__", None)
256            if mod_all is not None:
257                filt = mod_all.__contains__
258            else:
259                filt = _all_submodules
260
261            # don't traverse path items we've seen before
262            path = [p for p in (getattr(module, "__path__", None) or []) if not seen(p)]
263
264            yield from walk_packages2(pkgutil.iter_modules(path, f"{mod.name}."), filt)
265
266
267def module_mtime(modulename: str) -> float | None:
268    """Returns the time the specified module file was last modified, or `None` if this cannot be determined.
269    The primary use of this is live-reloading modules on modification."""
270    try:
271        with mock_some_common_side_effects():
272            spec = importlib.util.find_spec(modulename)
273    except AnyException:
274        pass
275    else:
276        if spec is not None and spec.origin is not None:
277            return Path(spec.origin).stat().st_mtime
278    return None
279
280
281def invalidate_caches(module_name: str) -> None:
282    """
283    Invalidate module cache to allow live-reloading of modules.
284    """
285    # Getting this right is tricky – reloading modules causes a bunch of surprising side effects.
286    # Our current best effort is to call `importlib.reload` on all modules that start with module_name.
287    # We also exclude our own dependencies, which cause fun errors otherwise.
288    if module_name not in sys.modules:
289        return
290    if any(
291        module_name.startswith(f"{x}.") or x == module_name
292        for x in ("jinja2", "markupsafe", "markdown2", "pygments")
293    ):
294        return
295
296    # a more extreme alternative:
297    # filename = sys.modules[module_name].__file__
298    # if (
299    #    filename.startswith(sysconfig.get_path("platstdlib"))
300    #    or filename.startswith(sysconfig.get_path("stdlib"))
301    # ):
302    #     return
303
304    importlib.invalidate_caches()
305    linecache.clearcache()
306    pdoc.doc.Module.from_name.cache_clear()
307    pdoc.doc_ast._get_source.cache_clear()
308    pdoc.docstrings.convert.cache_clear()
309
310    prefix = f"{module_name}."
311    mods = sorted(
312        mod for mod in sys.modules if module_name == mod or mod.startswith(prefix)
313    )
314    for modname in mods:
315        if modname == "pdoc.render":
316            # pdoc.render is stateful after configure(), so we don't want to reload it.
317            continue
318        try:
319            if not isinstance(sys.modules[modname], types.ModuleType):
320                continue  # some funky stuff going on - one example is typing.io, which is a class.
321            with mock_some_common_side_effects():
322                importlib.reload(sys.modules[modname])
323        except AnyException:
324            warnings.warn(
325                f"Error reloading {modname}:\n{traceback.format_exc()}",
326                stacklevel=2,
327            )