pdoc.render_helpers

  1from __future__ import annotations
  2
  3from collections.abc import Collection
  4from collections.abc import Iterable
  5from collections.abc import Mapping
  6from contextlib import contextmanager
  7import html
  8import inspect
  9import os
 10import re
 11from unittest.mock import patch
 12import warnings
 13
 14from jinja2 import ext
 15from jinja2 import nodes
 16import pygments.formatters
 17import pygments.lexers
 18
 19try:
 20    # Jinja2 >= 3.0
 21    from jinja2 import pass_context  # type: ignore
 22except ImportError:  # pragma: no cover
 23    from jinja2 import contextfilter as pass_context  # type: ignore
 24
 25from jinja2.runtime import Context
 26from markupsafe import Markup
 27
 28import pdoc.markdown2
 29
 30from . import docstrings
 31from ._compat import cache
 32from ._compat import removesuffix
 33
 34lexer = pygments.lexers.PythonLexer()
 35"""
 36The pygments lexer used for pdoc.render_helpers.highlight.
 37Overwrite this to configure pygments lexing.
 38"""
 39
 40formatter = pygments.formatters.HtmlFormatter(
 41    cssclass="pdoc-code codehilite",
 42    linenos="inline",
 43    anchorlinenos=True,
 44)
 45"""
 46The pygments formatter used for pdoc.render_helpers.highlight.
 47Overwrite this to configure pygments highlighting of code blocks.
 48
 49The usage of the `.codehilite` CSS selector in custom templates is deprecated since pdoc 10, use `.pdoc-code` instead.
 50"""
 51
 52signature_formatter = pygments.formatters.HtmlFormatter(nowrap=True)
 53"""
 54The pygments formatter used for pdoc.render_helpers.format_signature.
 55Overwrite this to configure pygments highlighting of signatures.
 56"""
 57
 58# Keep in sync with the documentation in pdoc/__init__.py!
 59markdown_extensions = {
 60    "code-friendly": None,
 61    "cuddled-lists": None,
 62    "fenced-code-blocks": {"cssclass": formatter.cssclass},
 63    "footnotes": None,
 64    "header-ids": None,
 65    "link-patterns": None,
 66    "markdown-in-html": None,
 67    "mermaid": None,
 68    "pyshell": None,
 69    "strike": None,
 70    "tables": None,
 71    "task_list": None,
 72    "toc": {"depth": 2},
 73}
 74"""
 75The default extensions loaded for `markdown2`.
 76Overwrite this to configure Markdown rendering.
 77"""
 78markdown_link_patterns = [
 79    (
 80        re.compile(
 81            r"""
 82            \b
 83            (
 84                (?:https?://|(?<!//)www\.)    # prefix - https:// or www.
 85                \w[\w_\-]*(?:\.\w[\w_\-]*)*   # host
 86                [^<>\s"']*                    # rest of url
 87                (?<![?!.,:*_~);])             # exclude trailing punctuation
 88                (?=[?!.,:*_~);]?(?:[<\s]|$))  # make sure that we're not followed by " or ', i.e. we're outside of href="...".
 89            )
 90        """,
 91            re.X,
 92        ),
 93        r"\1",
 94    )
 95]
 96"""
 97Link pattern used for markdown2's [`link-patterns` extra](https://github.com/trentm/python-markdown2/wiki/link-patterns).
 98"""
 99
100
101@cache
102def highlight(doc: pdoc.doc.Doc) -> str:
103    """Highlight the source code of a documentation object using pygments."""
104    if isinstance(doc, str):  # pragma: no cover
105        warnings.warn(
106            "Passing a string to the `highlight` render helper is deprecated, pass a pdoc.doc.Doc object instead.",
107            DeprecationWarning,
108        )
109        return Markup(pygments.highlight(doc, lexer, formatter))
110
111    # set up correct line numbers and anchors
112    formatter.linespans = doc.qualname or "L"
113    formatter.linenostart = doc.source_lines[0] + 1 if doc.source_lines else 1
114    return Markup(pygments.highlight(doc.source, lexer, formatter))
115
116
117def format_signature(sig: inspect.Signature, colon: bool) -> str:
118    """Format and highlight a function signature using pygments. Returns HTML."""
119    # First get a list with all params as strings.
120    result = pdoc.doc._PrettySignature._params(sig)  # type: ignore
121    return_annot = pdoc.doc._PrettySignature._return_annotation_str(sig)  # type: ignore
122
123    multiline = (
124        sum(len(x) + 2 for x in result) + len(return_annot)
125        > pdoc.doc._PrettySignature.MULTILINE_CUTOFF
126    )
127
128    def _try_highlight(code: str) -> str:
129        """Try to highlight a piece of code using pygments, but return the input as-is if pygments detects errors."""
130        pretty = pygments.highlight(code, lexer, signature_formatter).strip()
131        if '<span class="err">' not in pretty:
132            return pretty
133        else:
134            return html.escape(code)
135
136    # Next, individually highlight each parameter using pygments and wrap it in a span.param.
137    # This later allows us to properly control line breaks.
138    pretty_result = []
139    for i, param in enumerate(result):
140        pretty = _try_highlight(param)
141        if multiline:
142            pretty = f"""<span class="param">\t{pretty},</span>"""
143        else:
144            pretty = f"""<span class="param">{pretty}, </span>"""
145        pretty_result.append(pretty)
146
147    # remove last comma.
148    if pretty_result:
149        pretty_result[-1] = pretty_result[-1].rpartition(",")[0] + "</span>"
150
151    # Add return annotation.
152    anno = ")"
153    if return_annot:
154        anno += f" -> {_try_highlight(return_annot)}"
155    if colon:
156        anno += ":"
157    if return_annot or colon:
158        anno = f'<span class="return-annotation">{anno}</span>'
159
160    rendered = "(" + "".join(pretty_result) + anno
161
162    if multiline:
163        rendered = f'<span class="signature pdoc-code multiline">{rendered}</span>'
164    else:
165        rendered = f'<span class="signature pdoc-code condensed">{rendered}</span>'
166
167    return Markup(rendered)
168
169
170@cache
171def to_html(docstring: str) -> str:
172    """
173    Convert `docstring` from Markdown to HTML.
174    """
175    # careful: markdown2 returns a subclass of str with an extra
176    # .toc_html attribute. don't further process the result,
177    # otherwise this attribute will be lost.
178    return pdoc.markdown2.markdown(  # type: ignore
179        docstring,
180        extras=markdown_extensions,
181        link_patterns=markdown_link_patterns,
182    )
183
184
185@pass_context
186def to_markdown_with_context(context: Context, docstring: str) -> str:
187    """
188    Converts `docstring` from a custom docformat to Markdown (if necessary), and then from Markdown to HTML.
189    """
190    module: pdoc.doc.Module = context["module"]
191    docformat: str = context["docformat"]
192    return to_markdown(docstring, module, docformat)
193
194
195def to_markdown(docstring: str, module: pdoc.doc.Module, default_docformat: str) -> str:
196    docformat = getattr(module.obj, "__docformat__", default_docformat) or ""
197    return docstrings.convert(docstring, docformat, module.source_file)
198
199
200def possible_sources(
201    all_modules: Collection[str], identifier: str
202) -> Iterable[tuple[str, str]]:
203    """
204    For a given identifier, return all possible sources where it could originate from.
205    For example, assume `examplepkg._internal.Foo` with all_modules=["examplepkg"].
206    This could be a Foo class in _internal.py, or a nested `class _internal: class Foo` in examplepkg.
207    We return both candidates as we don't know if _internal.py exists.
208    It may not be in all_modules because it's been excluded by `__all__`.
209    However, if `examplepkg._internal` is in all_modules we know that it can only be that option.
210    """
211    if identifier in all_modules:
212        yield identifier, ""
213        return
214
215    modulename = identifier
216    qualname = None
217    while modulename:
218        modulename, _, add = modulename.rpartition(".")
219        qualname = f"{add}.{qualname}" if qualname else add
220        yield modulename, qualname
221        if modulename in all_modules:
222            return
223    raise ValueError(f"Invalid identifier: {identifier}")
224
225
226def split_identifier(all_modules: Collection[str], fullname: str) -> tuple[str, str]:
227    """
228    Split an identifier into a `(modulename, qualname)` tuple. For example, `pdoc.render_helpers.split_identifier`
229    would be split into `("pdoc.render_helpers","split_identifier")`. This is necessary to generate links to the
230    correct module.
231    """
232    warnings.warn(
233        "pdoc.render_helpers.split_identifier is deprecated and will be removed in a future release. "
234        "Use pdoc.render_helpers.possible_sources instead.",
235        DeprecationWarning,
236    )
237    *_, last = possible_sources(all_modules, fullname)
238    return last
239
240
241def _relative_link(current: list[str], target: list[str]) -> str:
242    if target == current:
243        return f"../{target[-1]}.html"
244    elif target[: len(current)] == current:
245        return "/".join(target[len(current) :]) + ".html"
246    else:
247        return "../" + _relative_link(current[:-1], target)
248
249
250@cache
251def relative_link(current_module: str, target_module: str) -> str:
252    """Compute the relative link to another module's HTML file."""
253    if current_module == target_module:
254        return ""
255    return _relative_link(
256        current_module.split(".")[:-1],
257        target_module.split("."),
258    )
259
260
261def qualname_candidates(identifier: str, context_qualname: str) -> list[str]:
262    """
263    Given an identifier in a current namespace, return all possible qualnames in the current module.
264    For example, if we are in Foo's subclass Bar and `baz()` is the identifier,
265    return `Foo.Bar.baz()`, `Foo.baz()`, and `baz()`.
266    """
267    end = len(context_qualname)
268    ret = []
269    while end > 0:
270        ret.append(f"{context_qualname[:end]}.{identifier}")
271        end = context_qualname.rfind(".", 0, end)
272    ret.append(identifier)
273    return ret
274
275
276@pass_context
277def linkify(context: Context, code: str, namespace: str = "") -> str:
278    """
279    Link all identifiers in a block of text. Identifiers referencing unknown modules or modules that
280    are not rendered at the moment will be ignored.
281    A piece of text is considered to be an identifier if it either contains a `.` or is surrounded by `<code>` tags.
282    """
283
284    def linkify_repl(m: re.Match):
285        text = m.group(0)
286        plain_text = text.replace(
287            '</span><span class="o">.</span><span class="n">', "."
288        )
289        identifier = removesuffix(plain_text, "()")
290        mod: pdoc.doc.Module = context["module"]
291
292        # Check if this is a relative reference?
293        if identifier.startswith("."):
294            taken_from_mod = mod
295            if namespace and (ns := mod.get(namespace)):
296                # Imported from somewhere else, so the relative reference should be from the original module.
297                taken_from_mod = context["all_modules"].get(ns.taken_from[0], mod)
298            if taken_from_mod.is_package:
299                # If we are in __init__.py, we want `.foo` to refer to a child module.
300                parent_module = taken_from_mod.modulename
301            else:
302                # If we are in a leaf module, we want `.foo` to refer to the adjacent module.
303                parent_module = taken_from_mod.modulename.rpartition(".")[0]
304            while identifier.startswith(".."):
305                identifier = identifier[1:]
306                parent_module = parent_module.rpartition(".")[0]
307            identifier = parent_module + identifier
308        else:
309            # Check if this is a local reference within this module?
310            for qualname in qualname_candidates(identifier, namespace):
311                doc = mod.get(qualname)
312                if doc and context["is_public"](doc).strip():
313                    return f'<a href="#{qualname}">{plain_text}</a>'
314
315        module = ""
316        qualname = ""
317        try:
318            # Check if the object we are interested in is imported and re-exposed in the current namespace.
319            for module, qualname in possible_sources(
320                context["all_modules"], identifier
321            ):
322                doc = mod.get(qualname)
323                if (
324                    doc
325                    and doc.taken_from == (module, qualname)
326                    and context["is_public"](doc).strip()
327                ):
328                    if plain_text.endswith("()"):
329                        plain_text = f"{doc.qualname}()"
330                    else:
331                        plain_text = doc.qualname
332                    return f'<a href="#{qualname}">{plain_text}</a>'
333        except ValueError:
334            # possible_sources did not find a parent module.
335            return text
336        else:
337            # It's not, but we now know the parent module. Does the target exist?
338            doc = context["all_modules"][module]
339            if qualname:
340                assert isinstance(doc, pdoc.doc.Module)
341                doc = doc.get(qualname)
342            target_exists_and_public = (
343                doc is not None and context["is_public"](doc).strip()
344            )
345            if target_exists_and_public:
346                assert doc is not None  # mypy
347                if qualname:
348                    qualname = f"#{qualname}"
349                if plain_text.endswith("()"):
350                    plain_text = f"{doc.fullname}()"
351                else:
352                    plain_text = doc.fullname
353                return f'<a href="{relative_link(context["module"].modulename, module)}{qualname}">{plain_text}</a>'
354            else:
355                return text
356
357    return Markup(
358        re.sub(
359            r"""
360            # Part 1: foo.bar or foo.bar() (without backticks)
361            (?<![/=?#&])  # heuristic: not part of a URL
362            # First part of the identifier (e.g. "foo") - this is optional for relative references.
363            (?:
364                \b
365                (?!\d)[a-zA-Z0-9_]+
366                |
367                \.*  # We may also start with multiple dots.
368            )
369            # Rest of the identifier (e.g. ".bar" or "..bar")
370            (?:
371                # A single dot or a dot surrounded with pygments highlighting.
372                (?:\.|</span><span\ class="o">\.</span><span\ class="n">)
373                (?!\d)[a-zA-Z0-9_]+
374            )+
375            (?:\(\)|\b(?!\(\)))  # we either end on () or on a word boundary.
376            (?!</a>)  # not an existing link
377            (?![/#])  # heuristic: not part of a URL
378
379            | # Part 2: `foo` or `foo()`. `foo.bar` is already covered with part 1.
380            (?<=<code>)
381                 (?!\d)[a-zA-Z0-9_]+
382            (?:\(\))?
383            (?=</code>(?!</a>))
384            """,
385            linkify_repl,
386            code,
387            flags=re.VERBOSE,
388        )
389    )
390
391
392@pass_context
393def link(context: Context, spec: tuple[str, str], text: str | None = None) -> str:
394    """Create a link for a specific `(modulename, qualname)` tuple."""
395    mod: pdoc.doc.Module = context["module"]
396    modulename, qualname = spec
397
398    # Check if the object we are interested is also imported and re-exposed in the current namespace.
399    # https://github.com/mitmproxy/pdoc/issues/490: We need to do this for every level, not just the tail.
400    doc: pdoc.doc.Doc | None = mod
401    for part in qualname.split("."):
402        doc = doc.get(part) if isinstance(doc, pdoc.doc.Namespace) else None
403        if not (
404            doc
405            and doc.taken_from[0] == modulename
406            and context["is_public"](doc).strip()
407        ):
408            break
409    else:
410        # everything down to the tail is imported and re-exposed.
411        if text:
412            text = text.replace(f"{modulename}.", f"{mod.modulename}.")
413        modulename = mod.modulename
414
415    if mod.modulename == modulename:
416        fullname = qualname
417    else:
418        fullname = removesuffix(f"{modulename}.{qualname}", ".")
419
420    if qualname:
421        qualname = f"#{qualname}"
422    if modulename in context["all_modules"]:
423        return Markup(
424            f'<a href="{relative_link(context["module"].modulename, modulename)}{qualname}">{text or fullname}</a>'
425        )
426    return text or fullname
427
428
429def edit_url(
430    modulename: str, is_package: bool, mapping: Mapping[str, str]
431) -> str | None:
432    """Create a link to edit a particular file in the used version control system."""
433    for m, prefix in mapping.items():
434        if m == modulename or modulename.startswith(f"{m}."):
435            filename = modulename[len(m) + 1 :].replace(".", "/")
436            if is_package:
437                filename = f"{filename}/__init__.py".lstrip("/")
438            else:
439                filename += ".py"
440            return f"{prefix}{filename}"
441    return None
442
443
444def root_module_name(all_modules: Mapping[str, pdoc.doc.Module]) -> str | None:
445    """
446    Return the name of the (unique) top-level module, or `None`
447    if no such module exists.
448
449    For example, assuming `foo`, `foo.bar`, and `foo.baz` are documented,
450    this function will return `foo`. If `foo` and `bar` are documented,
451    this function will return `None` as there is no unique top-level module.
452    """
453    shortest_name = min(all_modules, key=len, default=None)
454    prefix = f"{shortest_name}."
455    all_others_are_submodules = all(
456        x.startswith(prefix) or x == shortest_name for x in all_modules
457    )
458    if all_others_are_submodules:
459        return shortest_name
460    else:
461        return None
462
463
464def minify_css(css: str) -> str:
465    """Do some very basic CSS minification."""
466    css = re.sub(r"[ ]{4}|\n|(?<=[:{}]) | (?=[{}])", "", css)
467    css = re.sub(
468        r"/\*.+?\*/", lambda m: m.group(0) if m.group(0).startswith("/*!") else "", css
469    )
470    return Markup(css.replace("<style", "\n<style"))
471
472
473@contextmanager
474def defuse_unsafe_reprs():
475    """This decorator is applied by pdoc before calling an object's repr().
476    It applies some heuristics to patch our sensitive information.
477    For example, `os.environ`'s default `__repr__` implementation exposes all
478    local secrets.
479    """
480    with patch.object(os._Environ, "__repr__", lambda self: "os.environ"):
481        yield
482
483
484class DefaultMacroExtension(ext.Extension):
485    """
486    This extension provides a new `{% defaultmacro %}` statement, which defines a macro only if it does not exist.
487
488    For example,
489
490    ```html+jinja
491    {% defaultmacro example() %}
492        test 123
493    {% enddefaultmacro %}
494    ```
495
496    is equivalent to
497
498    ```html+jinja
499    {% macro default_example() %}
500    test 123
501    {% endmacro %}
502    {% if not example %}
503        {% macro example() %}
504            test 123
505        {% endmacro %}
506    {% endif %}
507    ```
508
509    Additionally, the default implementation is also available as `default_$macroname`, which makes it possible
510    to reference it in the override.
511    """
512
513    tags = {"defaultmacro"}
514
515    def parse(self, parser):
516        m = nodes.Macro(lineno=next(parser.stream).lineno)
517        name = parser.parse_assign_target(name_only=True).name
518        m.name = f"default_{name}"
519        parser.parse_signature(m)
520        m.body = parser.parse_statements(("name:enddefaultmacro",), drop_needle=True)
521
522        if_stmt = nodes.If(
523            nodes.Not(nodes.Name(name, "load")),
524            [nodes.Macro(name, m.args, m.defaults, m.body)],
525            [],
526            [],
527        )
528        return [m, if_stmt]