pdoc.doc

   1"""
   2This module defines pdoc's documentation objects. A documentation object corresponds to *something*
   3in your Python code that has a docstring or type annotation. Typically, this only includes
   4modules, classes, functions and methods. However, `pdoc` adds support for extracting documentation
   5from the abstract syntax tree, which means that variables (module, class or instance) are supported too.
   6
   7There are four main types of documentation objects:
   8
   9- `Module`
  10- `Class`
  11- `Function`
  12- `Variable`
  13
  14All documentation types make heavy use of `@functools.cached_property` decorators.
  15This means they have a large set of attributes that are lazily computed on first access.
  16By convention, all attributes are read-only, although this is not enforced at runtime.
  17"""
  18
  19from __future__ import annotations
  20
  21from abc import ABCMeta
  22from abc import abstractmethod
  23from collections.abc import Callable
  24import dataclasses
  25import enum
  26from functools import cache
  27from functools import cached_property
  28from functools import singledispatchmethod
  29from functools import wraps
  30import inspect
  31import os
  32from pathlib import Path
  33import re
  34import sys
  35import textwrap
  36import traceback
  37import types
  38from typing import Any
  39from typing import ClassVar
  40from typing import Generic
  41from typing import TypedDict
  42from typing import TypeVar
  43from typing import Union
  44from typing import get_origin
  45import warnings
  46
  47from pdoc import doc_ast
  48from pdoc import doc_pyi
  49from pdoc import extract
  50from pdoc._compat import TypeAlias
  51from pdoc._compat import TypeAliasType
  52from pdoc._compat import formatannotation
  53from pdoc._compat import is_typeddict
  54from pdoc.doc_types import GenericAlias
  55from pdoc.doc_types import NonUserDefinedCallables
  56from pdoc.doc_types import empty
  57from pdoc.doc_types import resolve_annotations
  58from pdoc.doc_types import safe_eval_type
  59
  60
  61def _include_fullname_in_traceback(f):
  62    """
  63    Doc.__repr__ should not raise, but it may raise if we screwed up.
  64    Debugging this is a bit tricky, because, well, we can't repr() in the traceback either then.
  65    This decorator adds location information to the traceback, which helps tracking down bugs.
  66    """
  67
  68    @wraps(f)
  69    def wrapper(self):
  70        try:
  71            return f(self)
  72        except Exception as e:
  73            raise RuntimeError(f"Error in {self.fullname}'s repr!") from e
  74
  75    return wrapper
  76
  77
  78T = TypeVar("T")
  79
  80
  81class Doc(Generic[T]):
  82    """
  83    A base class for all documentation objects.
  84    """
  85
  86    modulename: str
  87    """
  88    The module that this object is in, for example `pdoc.doc`.
  89    """
  90
  91    qualname: str
  92    """
  93    The qualified identifier name for this object. For example, if we have the following code:
  94    
  95    ```python
  96    class Foo:
  97        def bar(self):
  98            pass
  99    ```
 100    
 101    The qualname of `Foo`'s `bar` method is `Foo.bar`. The qualname of the `Foo` class is just `Foo`.
 102    
 103    See <https://www.python.org/dev/peps/pep-3155/> for details.
 104    """
 105
 106    obj: T
 107    """
 108    The underlying Python object.
 109    """
 110
 111    taken_from: tuple[str, str]
 112    """
 113    `(modulename, qualname)` of this doc object's original location.
 114    In the context of a module, this points to the location it was imported from,
 115    in the context of classes, this points to the class an attribute is inherited from.
 116    """
 117
 118    kind: ClassVar[str]
 119    """
 120    The type of the doc object, either `"module"`, `"class"`, `"function"`, or `"variable"`.
 121    """
 122
 123    @property
 124    def type(self) -> str:  # pragma: no cover
 125        warnings.warn(
 126            "pdoc.doc.Doc.type is deprecated. Use pdoc.doc.Doc.kind instead.",
 127            DeprecationWarning,
 128        )
 129        return self.kind
 130
 131    def __init__(
 132        self, modulename: str, qualname: str, obj: T, taken_from: tuple[str, str]
 133    ):
 134        """
 135        Initializes a documentation object, where
 136        `modulename` is the name this module is defined in,
 137        `qualname` contains a dotted path leading to the object from the module top-level, and
 138        `obj` is the object to document.
 139        """
 140        self.modulename = modulename
 141        self.qualname = qualname
 142        self.obj = obj
 143        self.taken_from = taken_from
 144
 145    @cached_property
 146    def fullname(self) -> str:
 147        """The full qualified name of this doc object, for example `pdoc.doc.Doc`."""
 148        # qualname is empty for modules
 149        return f"{self.modulename}.{self.qualname}".rstrip(".")
 150
 151    @cached_property
 152    def name(self) -> str:
 153        """The name of this object. For top-level functions and classes, this is equal to the qualname attribute."""
 154        return self.fullname.split(".")[-1]
 155
 156    @cached_property
 157    def docstring(self) -> str:
 158        """
 159        The docstring for this object. It has already been cleaned by `inspect.cleandoc`.
 160
 161        If no docstring can be found, an empty string is returned.
 162        """
 163        return _safe_getdoc(self.obj)
 164
 165    @cached_property
 166    def source(self) -> str:
 167        """
 168        The source code of the Python object as a `str`.
 169
 170        If the source cannot be obtained (for example, because we are dealing with a native C object),
 171        an empty string is returned.
 172        """
 173        return doc_ast.get_source(self.obj)
 174
 175    @cached_property
 176    def source_file(self) -> Path | None:
 177        """The name of the Python source file in which this object was defined. `None` for built-in objects."""
 178        try:
 179            return Path(inspect.getsourcefile(self.obj) or inspect.getfile(self.obj))  # type: ignore
 180        except TypeError:
 181            return None
 182
 183    @cached_property
 184    def source_lines(self) -> tuple[int, int] | None:
 185        """
 186        Return a `(start, end)` line number tuple for this object.
 187
 188        If no source file can be found, `None` is returned.
 189        """
 190        try:
 191            lines, start = inspect.getsourcelines(self.obj)  # type: ignore
 192            return start, start + len(lines) - 1
 193        except Exception:
 194            return None
 195
 196    @cached_property
 197    def is_inherited(self) -> bool:
 198        """
 199        If True, the doc object is inherited from another location.
 200        This most commonly refers to methods inherited by a subclass,
 201        but can also apply to variables that are assigned a class defined
 202        in a different module.
 203        """
 204        return (self.modulename, self.qualname) != self.taken_from
 205
 206    def __lt__(self, other):
 207        assert isinstance(other, Doc)
 208        return self.fullname.replace("__init__", "").__lt__(
 209            other.fullname.replace("__init__", "")
 210        )
 211
 212
 213class Namespace(Doc[T], metaclass=ABCMeta):
 214    """
 215    A documentation object that can have children. In other words, either a module or a class.
 216    """
 217
 218    @cached_property
 219    @abstractmethod
 220    def _member_objects(self) -> dict[str, Any]:
 221        """
 222        A mapping from *all* public and private member names to their Python objects.
 223        """
 224
 225    @cached_property
 226    @abstractmethod
 227    def _var_docstrings(self) -> dict[str, str]:
 228        """A mapping from some member variable names to their docstrings."""
 229
 230    @cached_property
 231    @abstractmethod
 232    def _func_docstrings(self) -> dict[str, str]:
 233        """A mapping from some member function names to their raw (not processed by any @decorators) docstrings."""
 234
 235    @cached_property
 236    @abstractmethod
 237    def _var_annotations(self) -> dict[str, Any]:
 238        """A mapping from some member variable names to their type annotations."""
 239
 240    @abstractmethod
 241    def _taken_from(self, member_name: str, obj: Any) -> tuple[str, str]:
 242        """The location this member was taken from. If unknown, (modulename, qualname) is returned."""
 243
 244    @cached_property
 245    @abstractmethod
 246    def own_members(self) -> list[Doc]:
 247        """A list of all own (i.e. non-inherited) `members`."""
 248
 249    @cached_property
 250    def members(self) -> dict[str, Doc]:
 251        """A mapping from all members to their documentation objects.
 252
 253        This mapping includes private members; they are only filtered out as part of the template logic.
 254        Constructors for enums, dicts, and abstract base classes are not picked up unless they have a custom docstring.
 255        """
 256        members: dict[str, Doc] = {}
 257        for name, obj in self._member_objects.items():
 258            qualname = f"{self.qualname}.{name}".lstrip(".")
 259            taken_from = self._taken_from(name, obj)
 260            doc: Doc[Any]
 261
 262            is_classmethod = isinstance(obj, classmethod)
 263            is_property = (
 264                isinstance(obj, (property, cached_property))
 265                or
 266                # Python 3.9 - 3.10: @classmethod @property is allowed.
 267                is_classmethod
 268                and isinstance(obj.__func__, (property, cached_property))
 269            )
 270            if is_property:
 271                func = obj
 272                if is_classmethod:
 273                    func = obj.__func__
 274                if isinstance(func, property):
 275                    func = func.fget
 276                else:
 277                    assert isinstance(func, cached_property)
 278                    func = func.func
 279
 280                doc_f = Function(self.modulename, qualname, func, taken_from)
 281                doc = Variable(
 282                    self.modulename,
 283                    qualname,
 284                    docstring=doc_f.docstring,
 285                    annotation=doc_f.signature.return_annotation,
 286                    default_value=empty,
 287                    taken_from=taken_from,
 288                )
 289                doc.source = doc_f.source
 290                doc.source_file = doc_f.source_file
 291                doc.source_lines = doc_f.source_lines
 292            elif inspect.isroutine(obj):
 293                doc = Function(self.modulename, qualname, obj, taken_from)  # type: ignore
 294            elif (
 295                inspect.isclass(obj)
 296                and obj is not empty
 297                and not isinstance(obj, GenericAlias)
 298                and obj.__qualname__.rpartition(".")[2] == qualname.rpartition(".")[2]
 299            ):
 300                # `dict[str,str]` is a GenericAlias instance. We want to render type aliases as variables though.
 301                doc = Class(self.modulename, qualname, obj, taken_from)
 302            elif inspect.ismodule(obj):
 303                if os.environ.get("PDOC_SUBMODULES"):  # pragma: no cover
 304                    doc = Module.from_name(obj.__name__)
 305                else:
 306                    continue
 307            elif inspect.isdatadescriptor(obj):
 308                doc = Variable(
 309                    self.modulename,
 310                    qualname,
 311                    docstring=getattr(obj, "__doc__", None) or "",
 312                    annotation=self._var_annotations.get(name, empty),
 313                    default_value=empty,
 314                    taken_from=taken_from,
 315                )
 316            else:
 317                doc = Variable(
 318                    self.modulename,
 319                    qualname,
 320                    docstring="",
 321                    annotation=self._var_annotations.get(name, empty),
 322                    default_value=obj,
 323                    taken_from=taken_from,
 324                )
 325            if self._var_docstrings.get(name):
 326                doc.docstring = self._var_docstrings[name]
 327            if self._func_docstrings.get(name) and not doc.docstring:
 328                doc.docstring = self._func_docstrings[name]
 329            members[doc.name] = doc
 330
 331        if isinstance(self, Module):
 332            # quirk: doc_pyi expects .members to be set already
 333            self.members = members  # type: ignore
 334            doc_pyi.include_typeinfo_from_stub_files(self)
 335
 336        return members
 337
 338    @cached_property
 339    def _members_by_origin(self) -> dict[tuple[str, str], list[Doc]]:
 340        """A mapping from (modulename, qualname) locations to the attributes taken from that path"""
 341        locations: dict[tuple[str, str], list[Doc]] = {}
 342        for member in self.members.values():
 343            mod, qualname = member.taken_from
 344            parent_qualname = ".".join(qualname.rsplit(".", maxsplit=1)[:-1])
 345            locations.setdefault((mod, parent_qualname), [])
 346            locations[(mod, parent_qualname)].append(member)
 347        return locations
 348
 349    @cached_property
 350    def inherited_members(self) -> dict[tuple[str, str], list[Doc]]:
 351        """A mapping from (modulename, qualname) locations to the attributes inherited from that path"""
 352        return {
 353            k: v
 354            for k, v in self._members_by_origin.items()
 355            if k not in (self.taken_from, (self.modulename, self.qualname))
 356        }
 357
 358    @cached_property
 359    def flattened_own_members(self) -> list[Doc]:
 360        """
 361        A list of all documented members and their child classes, recursively.
 362        """
 363        flattened = []
 364        for x in self.own_members:
 365            flattened.append(x)
 366            if isinstance(x, Class):
 367                flattened.extend(
 368                    [cls for cls in x.flattened_own_members if isinstance(cls, Class)]
 369                )
 370        return flattened
 371
 372    @cache
 373    def get(self, identifier: str) -> Doc | None:
 374        """Returns the documentation object for a particular identifier, or `None` if the identifier cannot be found."""
 375        head, _, tail = identifier.partition(".")
 376        if tail:
 377            h = self.members.get(head, None)
 378            if isinstance(h, Class):
 379                return h.get(tail)
 380            return None
 381        else:
 382            return self.members.get(identifier, None)
 383
 384
 385class Module(Namespace[types.ModuleType]):
 386    """
 387    Representation of a module's documentation.
 388    """
 389
 390    def __init__(
 391        self,
 392        module: types.ModuleType,
 393    ):
 394        """
 395        Creates a documentation object given the actual
 396        Python module object.
 397        """
 398        super().__init__(module.__name__, "", module, (module.__name__, ""))
 399
 400    kind = "module"
 401
 402    @classmethod
 403    @cache
 404    def from_name(cls, name: str) -> Module:
 405        """Create a `Module` object by supplying the module's (full) name."""
 406        return cls(extract.load_module(name))
 407
 408    @cache
 409    @_include_fullname_in_traceback
 410    def __repr__(self):
 411        return f"<module {self.fullname}{_docstr(self)}{_children(self)}>"
 412
 413    @cached_property
 414    def is_package(self) -> bool:
 415        """
 416        `True` if the module is a package, `False` otherwise.
 417
 418        Packages are a special kind of module that may have submodules.
 419        Typically, this means that this file is in a directory named like the
 420        module with the name `__init__.py`.
 421        """
 422        return _safe_getattr(self.obj, "__path__", None) is not None
 423
 424    @cached_property
 425    def _var_docstrings(self) -> dict[str, str]:
 426        return doc_ast.walk_tree(self.obj).var_docstrings
 427
 428    @cached_property
 429    def _func_docstrings(self) -> dict[str, str]:
 430        return doc_ast.walk_tree(self.obj).func_docstrings
 431
 432    @cached_property
 433    def _var_annotations(self) -> dict[str, Any]:
 434        annotations = doc_ast.walk_tree(self.obj).annotations.copy()
 435        for k, v in _safe_getattr(self.obj, "__annotations__", {}).items():
 436            annotations[k] = v
 437
 438        return resolve_annotations(annotations, self.obj, None, self.fullname)
 439
 440    def _taken_from(self, member_name: str, obj: Any) -> tuple[str, str]:
 441        if obj is empty:
 442            return self.modulename, f"{self.qualname}.{member_name}".lstrip(".")
 443        if isinstance(obj, types.ModuleType):
 444            return obj.__name__, ""
 445
 446        mod = _safe_getattr(obj, "__module__", None)
 447        qual = _safe_getattr(obj, "__qualname__", None)
 448        if mod and qual and "<locals>" not in qual:
 449            return mod, qual
 450        else:
 451            # This might be wrong, but it's the best guess we have.
 452            return (mod or self.modulename), f"{self.qualname}.{member_name}".lstrip(
 453                "."
 454            )
 455
 456    @cached_property
 457    def own_members(self) -> list[Doc]:
 458        return list(self.members.values())
 459
 460    @cached_property
 461    def submodules(self) -> list[Module]:
 462        """A list of all (direct) submodules."""
 463        include: Callable[[str], bool]
 464        mod_all = _safe_getattr(self.obj, "__all__", False)
 465        if mod_all is not False:
 466            mod_all_pos = {name: i for i, name in enumerate(mod_all)}
 467            include = mod_all_pos.__contains__
 468        else:
 469
 470            def include(name: str) -> bool:
 471                # optimization: we don't even try to load modules starting with an underscore as they would not be
 472                # visible by default. The downside of this is that someone who overrides `is_public` will miss those
 473                # entries, the upsides are 1) better performance and 2) less warnings because of import failures
 474                # (think of OS-specific modules, e.g. _linux.py failing to import on Windows).
 475                return not name.startswith("_")
 476
 477        submodules: list[Module] = []
 478        for mod_name, mod in extract.iter_modules2(self.obj).items():
 479            if not include(mod_name):
 480                continue
 481            try:
 482                module = Module.from_name(mod.name)
 483            except RuntimeError:
 484                warnings.warn(f"Couldn't import {mod.name}:\n{traceback.format_exc()}")
 485                continue
 486            submodules.append(module)
 487
 488        if mod_all:
 489            submodules = sorted(submodules, key=lambda m: mod_all_pos[m.name])
 490
 491        return submodules
 492
 493    @cached_property
 494    def _ast_keys(self) -> set[str]:
 495        return (
 496            self._var_docstrings.keys()
 497            | self._func_docstrings.keys()
 498            | self._var_annotations.keys()
 499        )
 500
 501    @cached_property
 502    def _member_objects(self) -> dict[str, Any]:
 503        members = {}
 504
 505        all_: list[str] | None = _safe_getattr(self.obj, "__all__", None)
 506        if all_ is not None:
 507            for name in all_:
 508                if not isinstance(name, str):
 509                    # Gracefully handle the case where objects are directly specified in __all__.
 510                    name = _safe_getattr(name, "__name__", str(name))
 511                if name in self.obj.__dict__:
 512                    val = self.obj.__dict__[name]
 513                elif name in self._var_annotations:
 514                    val = empty
 515                else:
 516                    # this may be an unimported submodule, try importing.
 517                    # (https://docs.python.org/3/tutorial/modules.html#importing-from-a-package)
 518                    try:
 519                        val = extract.load_module(f"{self.modulename}.{name}")
 520                    except RuntimeError as e:
 521                        warnings.warn(
 522                            f"Found {name!r} in {self.modulename}.__all__, but it does not resolve: {e}"
 523                        )
 524                        val = empty
 525                members[name] = val
 526
 527        else:
 528            # Starting with Python 3.10, __annotations__ is created on demand,
 529            # so we make a copy here as obj.__dict__ is changed while we iterate over it.
 530            # Additionally, accessing self._ast_keys may lead to the execution of TYPE_CHECKING blocks,
 531            # which may also modify obj.__dict__. (https://github.com/mitmproxy/pdoc/issues/351)
 532            for name, obj in list(self.obj.__dict__.items()):
 533                # We already exclude everything here that is imported.
 534                obj_module = inspect.getmodule(obj)
 535                declared_in_this_module = self.obj.__name__ == _safe_getattr(
 536                    obj_module, "__name__", None
 537                )
 538                include_in_docs = declared_in_this_module or name in self._ast_keys
 539                if include_in_docs:
 540                    members[name] = obj
 541
 542            for name in self._var_docstrings:
 543                members.setdefault(name, empty)
 544            for name in self._var_annotations:
 545                members.setdefault(name, empty)
 546
 547            members, notfound = doc_ast.sort_by_source(self.obj, {}, members)
 548            members.update(notfound)
 549
 550        return members
 551
 552    @cached_property
 553    def variables(self) -> list[Variable]:
 554        """
 555        A list of all documented module level variables.
 556        """
 557        return [x for x in self.members.values() if isinstance(x, Variable)]
 558
 559    @cached_property
 560    def classes(self) -> list[Class]:
 561        """
 562        A list of all documented module level classes.
 563        """
 564        return [x for x in self.members.values() if isinstance(x, Class)]
 565
 566    @cached_property
 567    def functions(self) -> list[Function]:
 568        """
 569        A list of all documented module level functions.
 570        """
 571        return [x for x in self.members.values() if isinstance(x, Function)]
 572
 573
 574class Class(Namespace[type]):
 575    """
 576    Representation of a class's documentation.
 577    """
 578
 579    kind = "class"
 580
 581    @cache
 582    @_include_fullname_in_traceback
 583    def __repr__(self):
 584        return f"<{_decorators(self)}class {self.modulename}.{self.qualname}{_docstr(self)}{_children(self)}>"
 585
 586    @cached_property
 587    def docstring(self) -> str:
 588        doc = Doc.docstring.__get__(self)  # type: ignore
 589        if doc == dict.__doc__:
 590            # Don't display default docstring for dict subclasses (primarily TypedDict).
 591            return ""
 592        if doc in _Enum_default_docstrings:
 593            # Don't display default docstring for enum subclasses.
 594            return ""
 595        if dataclasses.is_dataclass(self.obj) and doc.startswith(self.obj.__name__):
 596            try:
 597                sig = inspect.signature(self.obj)
 598            except Exception:
 599                pass
 600            else:
 601                # from https://github.com/python/cpython/blob/3.10/Lib/dataclasses.py
 602                is_dataclass_with_default_docstring = doc == self.obj.__name__ + str(
 603                    sig
 604                ).replace(" -> None", "")
 605                if is_dataclass_with_default_docstring:
 606                    return ""
 607        return doc
 608
 609    @cached_property
 610    def _var_docstrings(self) -> dict[str, str]:
 611        docstrings: dict[str, str] = {}
 612        for cls in self._bases:
 613            for name, docstr in doc_ast.walk_tree(cls).var_docstrings.items():
 614                docstrings.setdefault(name, docstr)
 615        return docstrings
 616
 617    @cached_property
 618    def _func_docstrings(self) -> dict[str, str]:
 619        docstrings: dict[str, str] = {}
 620        for cls in self._bases:
 621            for name, docstr in doc_ast.walk_tree(cls).func_docstrings.items():
 622                docstrings.setdefault(name, docstr)
 623        return docstrings
 624
 625    @cached_property
 626    def _var_annotations(self) -> dict[str, type]:
 627        # this is a bit tricky: __annotations__ also includes annotations from parent classes,
 628        # but we need to execute them in the namespace of the parent class.
 629        # Our workaround for this is to walk the MRO backwards, and only update/evaluate only if the annotation changes.
 630        annotations: dict[
 631            str, tuple[Any, type]
 632        ] = {}  # attribute -> (annotation_unresolved, annotation_resolved)
 633        for cls in reversed(self._bases):
 634            cls_annotations = doc_ast.walk_tree(cls).annotations.copy()
 635            dynamic_annotations = _safe_getattr(cls, "__annotations__", None)
 636            if isinstance(dynamic_annotations, dict):
 637                for attr, unresolved_annotation in dynamic_annotations.items():
 638                    cls_annotations[attr] = unresolved_annotation
 639            cls_fullname = (
 640                _safe_getattr(cls, "__module__", "") + "." + cls.__qualname__
 641            ).lstrip(".")
 642
 643            new_annotations = {
 644                attr: unresolved_annotation
 645                for attr, unresolved_annotation in cls_annotations.items()
 646                if attr not in annotations
 647                or annotations[attr][0] is not unresolved_annotation
 648            }
 649            localns = _safe_getattr(cls, "__dict__", None)
 650            for attr, t in resolve_annotations(
 651                new_annotations, inspect.getmodule(cls), localns, cls_fullname
 652            ).items():
 653                annotations[attr] = (new_annotations[attr], t)
 654
 655        return {k: v[1] for k, v in annotations.items()}
 656
 657    @cached_property
 658    def _bases(self) -> tuple[type, ...]:
 659        orig_bases = _safe_getattr(self.obj, "__orig_bases__", ())
 660
 661        if is_typeddict(self.obj):
 662            if sys.version_info < (3, 12):  # pragma: no cover
 663                # TypedDicts on Python <3.12 have a botched __mro__. We need to fix it.
 664                return (self.obj, *orig_bases[:-1])
 665            else:
 666                # TypedDict on Python >=3.12 removes intermediate classes from __mro__,
 667                # so we use orig_bases to recover the full mro.
 668                while orig_bases and orig_bases[-1] is not TypedDict:
 669                    parent_bases = _safe_getattr(orig_bases[-1], "__orig_bases__", ())
 670                    if (
 671                        len(parent_bases) != 1 or parent_bases in orig_bases
 672                    ):  # sanity check that things look right
 673                        break  # pragma: no cover
 674                    orig_bases = (*orig_bases, parent_bases[0])
 675
 676        # __mro__ and __orig_bases__ differ between Python versions and special cases like TypedDict/NamedTuple.
 677        # This here is a pragmatic approximation of what we want.
 678        return (
 679            *(base for base in orig_bases if isinstance(base, type)),
 680            *self.obj.__mro__,
 681        )
 682
 683    @cached_property
 684    def _declarations(self) -> dict[str, tuple[str, str]]:
 685        decls: dict[str, tuple[str, str]] = {}
 686        for cls in self._bases:
 687            treeinfo = doc_ast.walk_tree(cls)
 688            for name in (
 689                treeinfo.var_docstrings.keys()
 690                | treeinfo.func_docstrings.keys()
 691                | treeinfo.annotations.keys()
 692            ):
 693                decls.setdefault(name, (cls.__module__, f"{cls.__qualname__}.{name}"))
 694            for name in cls.__dict__:
 695                decls.setdefault(name, (cls.__module__, f"{cls.__qualname__}.{name}"))
 696        if decls.get("__init__", None) == ("builtins", "object.__init__"):
 697            decls["__init__"] = (
 698                self.obj.__module__,
 699                f"{self.obj.__qualname__}.__init__",
 700            )
 701        return decls
 702
 703    def _taken_from(self, member_name: str, obj: Any) -> tuple[str, str]:
 704        try:
 705            return self._declarations[member_name]
 706        except KeyError:  # pragma: no cover
 707            # TypedDict botches __mro__ on Python <3.12 and may need special casing here.
 708            # One workaround is to also specify TypedDict as a base class, see pdoc.doc.Class._bases.
 709            warnings.warn(
 710                f"Cannot determine where {self.fullname}.{member_name} is taken from, assuming current file."
 711            )
 712            return self.modulename, f"{self.qualname}.{member_name}"
 713
 714    @cached_property
 715    def own_members(self) -> list[Doc]:
 716        members = self._members_by_origin.get((self.modulename, self.qualname), [])
 717        if self.taken_from != (self.modulename, self.qualname):
 718            # .taken_from may be != (self.modulename, self.qualname), for example when
 719            # a module re-exports a class from a private submodule.
 720            members += self._members_by_origin.get(self.taken_from, [])
 721        return members
 722
 723    @cached_property
 724    def _member_objects(self) -> dict[str, Any]:
 725        unsorted: dict[str, Any] = {}
 726        for cls in self._bases:
 727            for name, obj in cls.__dict__.items():
 728                unsorted.setdefault(name, obj)
 729        for name in self._var_docstrings:
 730            unsorted.setdefault(name, empty)
 731        for name in self._var_annotations:
 732            unsorted.setdefault(name, empty)
 733
 734        init_has_no_doc = unsorted.get("__init__", object.__init__).__doc__ in (
 735            None,
 736            object.__init__.__doc__,
 737        )
 738        if init_has_no_doc:
 739            if inspect.isabstract(self.obj):
 740                # Special case: We don't want to show constructors for abstract base classes unless
 741                # they have a custom docstring.
 742                del unsorted["__init__"]
 743            elif issubclass(self.obj, enum.Enum):
 744                # Special case: Do not show a constructor for enums. They are typically not constructed by users.
 745                # The alternative would be showing __new__, as __call__ is too verbose.
 746                del unsorted["__init__"]
 747            elif issubclass(self.obj, dict):
 748                # Special case: Do not show a constructor for dict subclasses.
 749                unsorted.pop(
 750                    "__init__", None
 751                )  # TypedDict subclasses may not have __init__.
 752            else:
 753                # Check if there's a helpful Metaclass.__call__ or Class.__new__. This dance is very similar to
 754                # https://github.com/python/cpython/blob/9feae41c4f04ca27fd2c865807a5caeb50bf4fc4/Lib/inspect.py#L2359-L2376
 755                call = _safe_getattr(type(self.obj), "__call__", None)
 756                custom_call_with_custom_docstring = (
 757                    call is not None
 758                    and not isinstance(call, NonUserDefinedCallables)
 759                    and call.__doc__ not in (None, object.__call__.__doc__)
 760                )
 761                if custom_call_with_custom_docstring:
 762                    unsorted["__init__"] = call
 763                else:
 764                    # Does our class define a custom __new__ method?
 765                    new = _safe_getattr(self.obj, "__new__", None)
 766                    custom_new_with_custom_docstring = (
 767                        new is not None
 768                        and not isinstance(new, NonUserDefinedCallables)
 769                        and new.__doc__ not in (None, object.__new__.__doc__)
 770                    )
 771                    if custom_new_with_custom_docstring:
 772                        unsorted["__init__"] = new
 773
 774        sorted: dict[str, Any] = {}
 775        for cls in self._bases:
 776            sorted, unsorted = doc_ast.sort_by_source(cls, sorted, unsorted)
 777        sorted.update(unsorted)
 778        return sorted
 779
 780    @cached_property
 781    def bases(self) -> list[tuple[str, str, str]]:
 782        """
 783        A list of all base classes, i.e. all immediate parent classes.
 784
 785        Each parent class is represented as a `(modulename, qualname, display_text)` tuple.
 786        """
 787        bases = []
 788        for x in _safe_getattr(self.obj, "__orig_bases__", self.obj.__bases__):
 789            if x is object:
 790                continue
 791            o = get_origin(x)
 792            if o:
 793                bases.append((o.__module__, o.__qualname__, str(x)))
 794            elif x.__module__ == self.modulename:
 795                bases.append((x.__module__, x.__qualname__, x.__qualname__))
 796            else:
 797                bases.append(
 798                    (x.__module__, x.__qualname__, f"{x.__module__}.{x.__qualname__}")
 799                )
 800        return bases
 801
 802    @cached_property
 803    def decorators(self) -> list[str]:
 804        """A list of all decorators the class is decorated with."""
 805        decorators = []
 806        for t in doc_ast.parse(self.obj).decorator_list:
 807            decorators.append(f"@{doc_ast.unparse(t)}")
 808        return decorators
 809
 810    @cached_property
 811    def class_variables(self) -> list[Variable]:
 812        """
 813        A list of all documented class variables in the class.
 814
 815        Class variables are variables that are explicitly annotated with `typing.ClassVar`.
 816        All other variables are treated as instance variables.
 817        """
 818        return [
 819            x
 820            for x in self.members.values()
 821            if isinstance(x, Variable) and x.is_classvar
 822        ]
 823
 824    @cached_property
 825    def instance_variables(self) -> list[Variable]:
 826        """
 827        A list of all instance variables in the class.
 828        """
 829        return [
 830            x
 831            for x in self.members.values()
 832            if isinstance(x, Variable) and not x.is_classvar
 833        ]
 834
 835    @cached_property
 836    def classmethods(self) -> list[Function]:
 837        """
 838        A list of all documented `@classmethod`s.
 839        """
 840        return [
 841            x
 842            for x in self.members.values()
 843            if isinstance(x, Function) and x.is_classmethod
 844        ]
 845
 846    @cached_property
 847    def staticmethods(self) -> list[Function]:
 848        """
 849        A list of all documented `@staticmethod`s.
 850        """
 851        return [
 852            x
 853            for x in self.members.values()
 854            if isinstance(x, Function) and x.is_staticmethod
 855        ]
 856
 857    @cached_property
 858    def methods(self) -> list[Function]:
 859        """
 860        A list of all documented methods in the class that are neither static- nor classmethods.
 861        """
 862        return [
 863            x
 864            for x in self.members.values()
 865            if isinstance(x, Function)
 866            and not x.is_staticmethod
 867            and not x.is_classmethod
 868        ]
 869
 870
 871if sys.version_info >= (3, 10):
 872    WrappedFunction = types.FunctionType | staticmethod | classmethod
 873else:  # pragma: no cover
 874    WrappedFunction = Union[types.FunctionType, staticmethod, classmethod]
 875
 876
 877class Function(Doc[types.FunctionType]):
 878    """
 879    Representation of a function's documentation.
 880
 881    This class covers all "flavors" of functions, for example it also
 882    supports `@classmethod`s or `@staticmethod`s.
 883    """
 884
 885    kind = "function"
 886
 887    wrapped: WrappedFunction
 888    """The original wrapped function (e.g., `staticmethod(func)`)"""
 889
 890    obj: types.FunctionType
 891    """The unwrapped "real" function."""
 892
 893    def __init__(
 894        self,
 895        modulename: str,
 896        qualname: str,
 897        func: WrappedFunction,
 898        taken_from: tuple[str, str],
 899    ):
 900        """Initialize a function's documentation object."""
 901        unwrapped: types.FunctionType
 902        if isinstance(func, (classmethod, staticmethod)):
 903            unwrapped = func.__func__  # type: ignore
 904        elif isinstance(func, singledispatchmethod):
 905            unwrapped = func.func  # type: ignore
 906        elif hasattr(func, "__wrapped__"):
 907            unwrapped = func.__wrapped__
 908        else:
 909            unwrapped = func
 910        super().__init__(modulename, qualname, unwrapped, taken_from)
 911        self.wrapped = func
 912
 913    @cache
 914    @_include_fullname_in_traceback
 915    def __repr__(self):
 916        if self.is_classmethod:
 917            t = "class"
 918        elif self.is_staticmethod:
 919            t = "static"
 920        elif self.qualname != _safe_getattr(self.obj, "__name__", None):
 921            t = "method"
 922        else:
 923            t = "function"
 924        return f"<{_decorators(self)}{t} {self.funcdef} {self.name}{self.signature}: ...{_docstr(self)}>"
 925
 926    @cached_property
 927    def docstring(self) -> str:
 928        doc = Doc.docstring.__get__(self)  # type: ignore
 929        if not doc:
 930            # inspect.getdoc fails for inherited @classmethods and unbound @property descriptors.
 931            # We now do an ugly dance to obtain the bound object instead,
 932            # that somewhat resembles what inspect._findclass is doing.
 933            cls = sys.modules.get(_safe_getattr(self.obj, "__module__", None), None)
 934            for name in _safe_getattr(self.obj, "__qualname__", "").split(".")[:-1]:
 935                cls = _safe_getattr(cls, name, None)
 936
 937            unbound = _safe_getattr(cls, "__dict__", {}).get(self.name)
 938            is_classmethod_property = isinstance(unbound, classmethod) and isinstance(
 939                unbound.__func__, (property, cached_property)
 940            )
 941            if not is_classmethod_property:
 942                # We choke on @classmethod @property, but that's okay because it's been deprecated with Python 3.11.
 943                # Directly accessing them would give us the return value, which has the wrong docstring.
 944                doc = _safe_getdoc(_safe_getattr(cls, self.name, None))
 945
 946        if doc == object.__init__.__doc__:
 947            # inspect.getdoc(Foo.__init__) returns the docstring, for object.__init__ if left undefined...
 948            return ""
 949        else:
 950            return doc
 951
 952    @cached_property
 953    def is_classmethod(self) -> bool:
 954        """
 955        `True` if this function is a `@classmethod`, `False` otherwise.
 956        """
 957        return isinstance(self.wrapped, classmethod)
 958
 959    @cached_property
 960    def is_staticmethod(self) -> bool:
 961        """
 962        `True` if this function is a `@staticmethod`, `False` otherwise.
 963        """
 964        return isinstance(self.wrapped, staticmethod)
 965
 966    @cached_property
 967    def decorators(self) -> list[str]:
 968        """A list of all decorators the function is decorated with."""
 969        decorators = []
 970        obj: types.FunctionType = self.obj  # type: ignore
 971        for t in doc_ast.parse(obj).decorator_list:
 972            decorators.append(f"@{doc_ast.unparse(t)}")
 973        return decorators
 974
 975    @cached_property
 976    def funcdef(self) -> str:
 977        """
 978        The string of keywords used to define the function, i.e. `"def"` or `"async def"`.
 979        """
 980        if inspect.iscoroutinefunction(self.obj) or inspect.isasyncgenfunction(
 981            self.obj
 982        ):
 983            return "async def"
 984        else:
 985            return "def"
 986
 987    @cached_property
 988    def signature(self) -> inspect.Signature:
 989        """
 990        The function's signature.
 991
 992        This usually returns an instance of `_PrettySignature`, a subclass of `inspect.Signature`
 993        that contains pdoc-specific optimizations. For example, long argument lists are split over multiple lines
 994        in repr(). Additionally, all types are already resolved.
 995
 996        If the signature cannot be determined, a placeholder Signature object is returned.
 997        """
 998        if self.obj is object.__init__:
 999            # there is a weird edge case were inspect.signature returns a confusing (self, /, *args, **kwargs)
1000            # signature for the default __init__ method.
1001            return inspect.Signature()
1002        try:
1003            sig = _PrettySignature.from_callable(self.obj)
1004        except Exception:
1005            return inspect.Signature(
1006                [inspect.Parameter("unknown", inspect.Parameter.POSITIONAL_OR_KEYWORD)]
1007            )
1008        mod = inspect.getmodule(self.obj)
1009        globalns = _safe_getattr(mod, "__dict__", {})
1010        localns = globalns
1011        for parent_cls_name in self.qualname.split(".")[:-1]:
1012            parent_cls = localns.get(parent_cls_name, object)
1013            localns = _safe_getattr(parent_cls, "__dict__", None)
1014            if localns is None:
1015                break  # pragma: no cover
1016
1017        if self.name == "__init__":
1018            sig = sig.replace(return_annotation=empty)
1019        else:
1020            sig = sig.replace(
1021                return_annotation=safe_eval_type(
1022                    sig.return_annotation, globalns, localns, mod, self.fullname
1023                )
1024            )
1025        for p in sig.parameters.values():
1026            p._annotation = safe_eval_type(  # type: ignore
1027                p.annotation, globalns, localns, mod, self.fullname
1028            )
1029        return sig
1030
1031    @cached_property
1032    def signature_without_self(self) -> inspect.Signature:
1033        """Like `signature`, but without the first argument.
1034
1035        This is useful to display constructors.
1036        """
1037        return self.signature.replace(
1038            parameters=list(self.signature.parameters.values())[1:]
1039        )
1040
1041
1042class Variable(Doc[None]):
1043    """
1044    Representation of a variable's documentation. This includes module, class and instance variables.
1045    """
1046
1047    kind = "variable"
1048
1049    default_value: (
1050        Any | empty
1051    )  # technically Any includes empty, but this conveys intent.
1052    """
1053    The variable's default value.
1054    
1055    In some cases, no default value is known. This may either be because a variable is only defined in the constructor,
1056    or it is only declared with a type annotation without assignment (`foo: int`).
1057    To distinguish this case from a default value of `None`, `pdoc.doc_types.empty` is used as a placeholder.
1058    """
1059
1060    annotation: type | empty
1061    """
1062    The variable's type annotation.
1063    
1064    If there is no type annotation, `pdoc.doc_types.empty` is used as a placeholder.
1065    """
1066
1067    def __init__(
1068        self,
1069        modulename: str,
1070        qualname: str,
1071        *,
1072        taken_from: tuple[str, str],
1073        docstring: str,
1074        annotation: type | empty = empty,
1075        default_value: Any | empty = empty,
1076    ):
1077        """
1078        Construct a variable doc object.
1079
1080        While classes and functions can introspect themselves to see their docstring,
1081        variables can't do that as we don't have a "variable object" we could query.
1082        As such, docstring, declaration location, type annotation, and the default value
1083        must be passed manually in the constructor.
1084        """
1085        super().__init__(modulename, qualname, None, taken_from)
1086        # noinspection PyPropertyAccess
1087        self.docstring = inspect.cleandoc(docstring)
1088        self.annotation = annotation
1089        self.default_value = default_value
1090
1091    @cache
1092    @_include_fullname_in_traceback
1093    def __repr__(self):
1094        if self.default_value_str:
1095            default = f" = {self.default_value_str}"
1096        else:
1097            default = ""
1098        return f'<var {self.qualname.rsplit(".")[-1]}{self.annotation_str}{default}{_docstr(self)}>'
1099
1100    @cached_property
1101    def is_classvar(self) -> bool:
1102        """`True` if the variable is a class variable, `False` otherwise."""
1103        if get_origin(self.annotation) is ClassVar:
1104            return True
1105        else:
1106            return False
1107
1108    @cached_property
1109    def is_typevar(self) -> bool:
1110        """`True` if the variable is a `typing.TypeVar`, `False` otherwise."""
1111        if isinstance(self.default_value, TypeVar):
1112            return True
1113        else:
1114            return False
1115
1116    @cached_property
1117    def is_type_alias_type(self) -> bool:
1118        """`True` if the variable is a `typing.TypeAliasType`, `False` otherwise."""
1119        return isinstance(self.default_value, TypeAliasType)
1120
1121    @cached_property
1122    def is_enum_member(self) -> bool:
1123        """`True` if the variable is an enum member, `False` otherwise."""
1124        if isinstance(self.default_value, enum.Enum):
1125            return True
1126        else:
1127            return False
1128
1129    @cached_property
1130    def default_value_str(self) -> str:
1131        """The variable's default value as a pretty-printed str."""
1132        if self.default_value is empty:
1133            return ""
1134        if isinstance(self.default_value, TypeAliasType):
1135            formatted = formatannotation(self.default_value.__value__)
1136            return _remove_collections_abc(formatted)
1137        elif self.annotation == TypeAlias:
1138            formatted = formatannotation(self.default_value)
1139            return _remove_collections_abc(formatted)
1140
1141        # This is not perfect, but a solid attempt at preventing accidental leakage of secrets.
1142        # If you have input on how to improve the heuristic, please send a pull request!
1143        value_taken_from_env_var = (
1144            isinstance(self.default_value, str)
1145            and len(self.default_value) >= 8
1146            and self.default_value in _environ_lookup()
1147        )
1148        if value_taken_from_env_var and not os.environ.get("PDOC_DISPLAY_ENV_VARS", ""):
1149            env_var = "$" + _environ_lookup()[self.default_value]
1150            warnings.warn(
1151                f"The default value of {self.fullname} matches the {env_var} environment variable. "
1152                f"To prevent accidental leakage of secrets, the default value is not displayed. "
1153                f"Disable this behavior by setting PDOC_DISPLAY_ENV_VARS=1 as an environment variable.",
1154                RuntimeWarning,
1155            )
1156            return env_var
1157
1158        try:
1159            pretty = repr(self.default_value)
1160        except Exception as e:
1161            warnings.warn(f"repr({self.fullname}) raised an exception ({e!r})")
1162            return ""
1163
1164        pretty = _remove_memory_addresses(pretty)
1165        return pretty
1166
1167    @cached_property
1168    def annotation_str(self) -> str:
1169        """The variable's type annotation as a pretty-printed str."""
1170        if self.annotation is not empty:
1171            formatted = formatannotation(self.annotation)
1172            return f": {_remove_collections_abc(formatted)}"
1173        else:
1174            return ""
1175
1176
1177@cache
1178def _environ_lookup():
1179    """
1180    A reverse lookup of os.environ. This is a cached function so that it is evaluated lazily.
1181    """
1182    return {value: key for key, value in os.environ.items()}
1183
1184
1185class _PrettySignature(inspect.Signature):
1186    """
1187    A subclass of `inspect.Signature` that pads __str__ over several lines
1188    for complex signatures.
1189    """
1190
1191    MULTILINE_CUTOFF = 70
1192
1193    def _params(self) -> list[str]:
1194        # redeclared here to keep code snipped below as-is.
1195        _POSITIONAL_ONLY = inspect.Parameter.POSITIONAL_ONLY
1196        _VAR_POSITIONAL = inspect.Parameter.VAR_POSITIONAL
1197        _KEYWORD_ONLY = inspect.Parameter.KEYWORD_ONLY
1198
1199        # https://github.com/python/cpython/blob/799f8489d418b7f9207d333eac38214931bd7dcc/Lib/inspect.py#L3083-L3117
1200        # Change: added re.sub() to formatted = ....
1201        # ✂ start ✂
1202        result = []
1203        render_pos_only_separator = False
1204        render_kw_only_separator = True
1205        for param in self.parameters.values():
1206            formatted = str(param)
1207            formatted = _remove_memory_addresses(formatted)
1208            formatted = _remove_collections_abc(formatted)
1209
1210            kind = param.kind
1211
1212            if kind == _POSITIONAL_ONLY:
1213                render_pos_only_separator = True
1214            elif render_pos_only_separator:
1215                # It's not a positional-only parameter, and the flag
1216                # is set to 'True' (there were pos-only params before.)
1217                result.append("/")
1218                render_pos_only_separator = False
1219
1220            if kind == _VAR_POSITIONAL:
1221                # OK, we have an '*args'-like parameter, so we won't need
1222                # a '*' to separate keyword-only arguments
1223                render_kw_only_separator = False
1224            elif kind == _KEYWORD_ONLY and render_kw_only_separator:
1225                # We have a keyword-only parameter to render and we haven't
1226                # rendered an '*args'-like parameter before, so add a '*'
1227                # separator to the parameters list ("foo(arg1, *, arg2)" case)
1228                result.append("*")
1229                # This condition should be only triggered once, so
1230                # reset the flag
1231                render_kw_only_separator = False
1232
1233            result.append(formatted)
1234
1235        if render_pos_only_separator:
1236            # There were only positional-only parameters, hence the
1237            # flag was not reset to 'False'
1238            result.append("/")
1239        # ✂ end ✂
1240
1241        return result
1242
1243    def _return_annotation_str(self) -> str:
1244        if self.return_annotation is not empty:
1245            formatted = formatannotation(self.return_annotation)
1246            return _remove_collections_abc(formatted)
1247        else:
1248            return ""
1249
1250    def __str__(self):
1251        result = self._params()
1252        return_annot = self._return_annotation_str()
1253
1254        total_len = sum(len(x) + 2 for x in result) + len(return_annot)
1255
1256        if total_len > self.MULTILINE_CUTOFF:
1257            rendered = "(\n    " + ",\n    ".join(result) + "\n)"
1258        else:
1259            rendered = "({})".format(", ".join(result))
1260        if return_annot:
1261            rendered += f" -> {return_annot}"
1262
1263        return rendered
1264
1265
1266def _cut(x: str) -> str:
1267    """helper function for Doc.__repr__()"""
1268    if len(x) < 20:
1269        return x
1270    else:
1271        return x[:20] + "…"
1272
1273
1274def _docstr(doc: Doc) -> str:
1275    """helper function for Doc.__repr__()"""
1276    docstr = []
1277    if doc.is_inherited:
1278        docstr.append(f"inherited from {'.'.join(doc.taken_from).rstrip('.')}")
1279    if doc.docstring:
1280        docstr.append(_cut(doc.docstring))
1281    if docstr:
1282        return f"  # {', '.join(docstr)}"
1283    else:
1284        return ""
1285
1286
1287def _decorators(doc: Class | Function) -> str:
1288    """helper function for Doc.__repr__()"""
1289    if doc.decorators:
1290        return " ".join(doc.decorators) + " "
1291    else:
1292        return ""
1293
1294
1295def _children(doc: Namespace) -> str:
1296    children = "\n".join(
1297        repr(x)
1298        for x in doc.members.values()
1299        if not x.name.startswith("_") or x.name == "__init__"
1300    )
1301    if children:
1302        children += "\n"
1303        children = f"\n{textwrap.indent(children, '    ')}"
1304    return children
1305
1306
1307def _safe_getattr(obj, attr, default):
1308    """Like `getattr()`, but never raises."""
1309    try:
1310        return getattr(obj, attr, default)
1311    except Exception as e:
1312        warnings.warn(
1313            f"getattr({obj!r}, {attr!r}, {default!r}) raised an exception: {e!r}"
1314        )
1315        return default
1316
1317
1318def _safe_getdoc(obj: Any) -> str:
1319    """Like `inspect.getdoc()`, but never raises. Always returns a stripped string."""
1320    try:
1321        doc = inspect.getdoc(obj) or ""
1322    except Exception as e:
1323        warnings.warn(f"inspect.getdoc({obj!r}) raised an exception: {e!r}")
1324        return ""
1325    else:
1326        return doc.strip()
1327
1328
1329_Enum_default_docstrings = tuple(
1330    {
1331        _safe_getdoc(enum.Enum),
1332        _safe_getdoc(enum.IntEnum),
1333        _safe_getdoc(_safe_getattr(enum, "StrEnum", enum.Enum)),
1334    }
1335)
1336
1337
1338def _remove_memory_addresses(x: str) -> str:
1339    """Remove memory addresses from repr() output"""
1340    return re.sub(r" at 0x[0-9a-fA-F]+(?=>)", "", x)
1341
1342
1343def _remove_collections_abc(x: str) -> str:
1344    """Remove 'collections.abc' from type signatures."""
1345    return re.sub(r"(?!\.)\bcollections\.abc\.", "", x)