pdoc.render_helpers

  1from __future__ import annotations
  2
  3from collections.abc import Collection
  4from collections.abc import Iterable
  5from collections.abc import Mapping
  6from contextlib import contextmanager
  7from functools import cache
  8import html
  9import inspect
 10import os
 11import re
 12from unittest.mock import patch
 13import warnings
 14
 15from jinja2 import ext
 16from jinja2 import nodes
 17import pygments.formatters
 18import pygments.lexers
 19
 20try:
 21    # Jinja2 >= 3.0
 22    from jinja2 import pass_context  # type: ignore
 23except ImportError:  # pragma: no cover
 24    from jinja2 import contextfilter as pass_context  # type: ignore
 25
 26from jinja2.runtime import Context
 27from markupsafe import Markup
 28
 29import pdoc.markdown2
 30
 31from . import docstrings
 32
 33lexer = pygments.lexers.PythonLexer()
 34"""
 35The pygments lexer used for pdoc.render_helpers.highlight.
 36Overwrite this to configure pygments lexing.
 37"""
 38
 39formatter = pygments.formatters.HtmlFormatter(
 40    cssclass="pdoc-code codehilite",
 41    linenos="inline",
 42    anchorlinenos=True,
 43)
 44"""
 45The pygments formatter used for pdoc.render_helpers.highlight.
 46Overwrite this to configure pygments highlighting of code blocks.
 47
 48The usage of the `.codehilite` CSS selector in custom templates is deprecated since pdoc 10, use `.pdoc-code` instead.
 49"""
 50
 51signature_formatter = pygments.formatters.HtmlFormatter(nowrap=True)
 52"""
 53The pygments formatter used for pdoc.render_helpers.format_signature.
 54Overwrite this to configure pygments highlighting of signatures.
 55"""
 56
 57# Keep in sync with the documentation in pdoc/__init__.py!
 58markdown_extensions = {
 59    "alerts": None,
 60    "code-friendly": None,
 61    "cuddled-lists": None,
 62    "fenced-code-blocks": {"cssclass": formatter.cssclass},
 63    "footnotes": None,
 64    "header-ids": None,
 65    "link-patterns": None,
 66    "markdown-in-html": None,
 67    "mermaid": None,
 68    "pyshell": None,
 69    "strike": None,
 70    "tables": None,
 71    "task_list": None,
 72    "toc": {"depth": 2},
 73}
 74"""
 75The default extensions loaded for `markdown2`.
 76Overwrite this to configure Markdown rendering.
 77"""
 78markdown_link_patterns = [
 79    (
 80        re.compile(
 81            r"""
 82            \b
 83            (
 84                (?:https?://|(?<!//)www\.)    # prefix - https:// or www.
 85                \w[\w_\-]*(?:\.\w[\w_\-]*)*   # host
 86                [^<>\s"']*                    # rest of url
 87                (?<![?!.,:*_~);])             # exclude trailing punctuation
 88                (?=[?!.,:*_~);]?(?:[<\s]|$))  # make sure that we're not followed by " or ', i.e. we're outside of href="...".
 89            )
 90        """,
 91            re.X,
 92        ),
 93        r"\1",
 94    )
 95]
 96"""
 97Link pattern used for markdown2's [`link-patterns` extra](https://github.com/trentm/python-markdown2/wiki/link-patterns).
 98"""
 99
100
101@cache
102def highlight(doc: pdoc.doc.Doc) -> str:
103    """Highlight the source code of a documentation object using pygments."""
104    if isinstance(doc, str):  # pragma: no cover
105        warnings.warn(
106            "Passing a string to the `highlight` render helper is deprecated, pass a pdoc.doc.Doc object instead.",
107            DeprecationWarning,
108        )
109        return Markup(pygments.highlight(doc, lexer, formatter))
110
111    # set up correct line numbers and anchors
112    formatter.linespans = doc.qualname or "L"
113    formatter.linenostart = doc.source_lines[0] + 1 if doc.source_lines else 1
114    return Markup(pygments.highlight(doc.source, lexer, formatter))
115
116
117def format_signature(sig: inspect.Signature, colon: bool) -> str:
118    """Format and highlight a function signature using pygments. Returns HTML."""
119    # First get a list with all params as strings.
120    result = pdoc.doc._PrettySignature._params(sig)  # type: ignore
121    return_annot = pdoc.doc._PrettySignature._return_annotation_str(sig)  # type: ignore
122
123    multiline = (
124        sum(len(x) + 2 for x in result) + len(return_annot)
125        > pdoc.doc._PrettySignature.MULTILINE_CUTOFF
126    )
127
128    def _try_highlight(code: str) -> str:
129        """Try to highlight a piece of code using pygments, but return the input as-is if pygments detects errors."""
130        pretty = pygments.highlight(code, lexer, signature_formatter).strip()
131        if '<span class="err">' not in pretty:
132            return pretty
133        else:
134            return html.escape(code)
135
136    # Next, individually highlight each parameter using pygments and wrap it in a span.param.
137    # This later allows us to properly control line breaks.
138    pretty_result = []
139    for i, param in enumerate(result):
140        pretty = _try_highlight(param)
141        if multiline:
142            pretty = f"""<span class="param">\t{pretty},</span>"""
143        else:
144            pretty = f"""<span class="param">{pretty}, </span>"""
145        pretty_result.append(pretty)
146
147    # remove last comma.
148    if pretty_result:
149        pretty_result[-1] = pretty_result[-1].rpartition(",")[0] + "</span>"
150
151    # Add return annotation.
152    anno = ")"
153    if return_annot:
154        anno += f" -> {_try_highlight(return_annot)}"
155    if colon:
156        anno += ":"
157    if return_annot or colon:
158        anno = f'<span class="return-annotation">{anno}</span>'
159
160    rendered = "(" + "".join(pretty_result) + anno
161
162    if multiline:
163        rendered = f'<span class="signature pdoc-code multiline">{rendered}</span>'
164    else:
165        rendered = f'<span class="signature pdoc-code condensed">{rendered}</span>'
166
167    return Markup(rendered)
168
169
170@cache
171def to_html(docstring: str) -> str:
172    """
173    Convert `docstring` from Markdown to HTML.
174    """
175    # careful: markdown2 returns a subclass of str with an extra
176    # .toc_html attribute. don't further process the result,
177    # otherwise this attribute will be lost.
178    return pdoc.markdown2.markdown(  # type: ignore
179        docstring,
180        extras=markdown_extensions,
181        link_patterns=markdown_link_patterns,
182    )
183
184
185@pass_context
186def to_markdown_with_context(context: Context, docstring: str) -> str:
187    """
188    Converts `docstring` from a custom docformat to Markdown (if necessary), and then from Markdown to HTML.
189    """
190    module: pdoc.doc.Module = context["module"]
191    docformat: str = context["docformat"]
192    return to_markdown(docstring, module, docformat)
193
194
195def to_markdown(docstring: str, module: pdoc.doc.Module, default_docformat: str) -> str:
196    docformat = getattr(module.obj, "__docformat__", default_docformat) or ""
197    return docstrings.convert(docstring, docformat, module.source_file)
198
199
200def possible_sources(
201    all_modules: Collection[str], identifier: str
202) -> Iterable[tuple[str, str]]:
203    """
204    For a given identifier, return all possible sources where it could originate from.
205    For example, assume `examplepkg._internal.Foo` with all_modules=["examplepkg"].
206    This could be a Foo class in _internal.py, or a nested `class _internal: class Foo` in examplepkg.
207    We return both candidates as we don't know if _internal.py exists.
208    It may not be in all_modules because it's been excluded by `__all__`.
209    However, if `examplepkg._internal` is in all_modules we know that it can only be that option.
210
211    >>> possible_sources(["examplepkg"], "examplepkg.Foo.bar")
212    examplepkg.Foo, bar
213    examplepkg, Foo.bar
214    """
215    if identifier in all_modules:
216        yield identifier, ""
217        return
218
219    modulename = identifier
220    qualname = None
221    while modulename:
222        modulename, _, add = modulename.rpartition(".")
223        qualname = f"{add}.{qualname}" if qualname else add
224        yield modulename, qualname
225        if modulename in all_modules:
226            return
227    raise ValueError(f"Invalid identifier: {identifier}")
228
229
230def split_identifier(all_modules: Collection[str], fullname: str) -> tuple[str, str]:
231    """
232    Split an identifier into a `(modulename, qualname)` tuple. For example, `pdoc.render_helpers.split_identifier`
233    would be split into `("pdoc.render_helpers","split_identifier")`. This is necessary to generate links to the
234    correct module.
235    """
236    warnings.warn(
237        "pdoc.render_helpers.split_identifier is deprecated and will be removed in a future release. "
238        "Use pdoc.render_helpers.possible_sources instead.",
239        DeprecationWarning,
240    )
241    *_, last = possible_sources(all_modules, fullname)
242    return last
243
244
245def _relative_link(current: list[str], target: list[str]) -> str:
246    if target == current:
247        return f"../{target[-1]}.html"
248    elif target[: len(current)] == current:
249        return "/".join(target[len(current) :]) + ".html"
250    else:
251        return "../" + _relative_link(current[:-1], target)
252
253
254@cache
255def relative_link(current_module: str, target_module: str) -> str:
256    """Compute the relative link to another module's HTML file."""
257    if current_module == target_module:
258        return ""
259    return _relative_link(
260        current_module.split(".")[:-1],
261        target_module.split("."),
262    )
263
264
265def qualname_candidates(identifier: str, context_qualname: str) -> list[str]:
266    """
267    Given an identifier in a current namespace, return all possible qualnames in the current module.
268    For example, if we are in Foo's subclass Bar and `baz()` is the identifier,
269    return `Foo.Bar.baz()`, `Foo.baz()`, and `baz()`.
270    """
271    end = len(context_qualname)
272    ret = []
273    while end > 0:
274        ret.append(f"{context_qualname[:end]}.{identifier}")
275        end = context_qualname.rfind(".", 0, end)
276    ret.append(identifier)
277    return ret
278
279
280def module_candidates(identifier: str, current_module: str) -> Iterable[str]:
281    """
282    Given an identifier and the current module name, return the module names we should look at
283    to find where the target object is exposed. Module names are ordered by preferences, i.e.
284    we always prefer the current module and then top-level modules over their children.
285
286    >>> module_candidates("foo.bar.baz", "qux")
287    qux
288    foo
289    foo.bar
290    foo.bar.baz
291    >>> module_candidates("foo.bar.baz", "foo.bar")
292    foo.bar
293    foo
294    foo.bar.baz
295    """
296    yield current_module
297
298    end = identifier.find(".")
299    while end > 0:
300        if (name := identifier[:end]) != current_module:
301            yield name
302        end = identifier.find(".", end + 1)
303
304    if identifier != current_module:
305        yield identifier
306
307
308@pass_context
309def linkify(
310    context: Context, code: str, namespace: str = "", shorten: bool = True
311) -> str:
312    """
313    Link all identifiers in a block of text. Identifiers referencing unknown modules or modules that
314    are not rendered at the moment will be ignored.
315    A piece of text is considered to be an identifier if it either contains a `.` or is surrounded by `<code>` tags.
316
317    If `shorten` is True, replace identifiers with short forms where possible.
318    For example, replace "current_module.Foo" with "Foo". This is useful for annotations
319    (which are verbose), but undesired for docstrings (where we want to preserve intent).
320    """
321
322    def linkify_repl(m: re.Match):
323        """
324        Resolve `text` to the most suitable documentation object.
325        """
326        text = m.group(0)
327        plain_text = text.replace(
328            '</span><span class="o">.</span><span class="n">', "."
329        )
330        identifier = plain_text.removesuffix("()")
331        mod: pdoc.doc.Module = context["module"]
332
333        # Check if this is a relative reference. These cannot be local and need to be resolved.
334        if identifier.startswith("."):
335            taken_from_mod = mod
336            if namespace and (ns := mod.get(namespace)):
337                # Imported from somewhere else, so the relative reference should be from the original module.
338                taken_from_mod = context["all_modules"].get(ns.taken_from[0], mod)
339            if taken_from_mod.is_package:
340                # If we are in __init__.py, we want `.foo` to refer to a child module.
341                parent_module = taken_from_mod.modulename
342            else:
343                # If we are in a leaf module, we want `.foo` to refer to the adjacent module.
344                parent_module = taken_from_mod.modulename.rpartition(".")[0]
345            while identifier.startswith(".."):
346                identifier = identifier[1:]
347                parent_module = parent_module.rpartition(".")[0]
348            identifier = parent_module + identifier
349        else:
350            # Is this a local reference within this module?
351            for qualname in qualname_candidates(identifier, namespace):
352                doc = mod.get(qualname)
353                if doc and context["is_public"](doc).strip():
354                    return f'<a href="#{qualname}">{plain_text}</a>'
355
356        # Is this a reference pointing straight at a module?
357        if identifier in context["all_modules"]:
358            return f'<a href="{relative_link(context["module"].modulename, identifier)}">{identifier}</a>'
359
360        try:
361            sources = list(possible_sources(context["all_modules"], identifier))
362        except ValueError:
363            # possible_sources did not find a parent module.
364            return text
365
366        # Try to find the actual target object so that we can then later verify
367        # that objects exposed at a parent module with the same name point to it.
368        target_object = None
369        for module_name, qualname in sources:
370            if doc := context["all_modules"].get(module_name, {}).get(qualname):
371                target_object = doc.obj
372                break
373
374        # Look at the different modules where our target object may be exposed.
375        for module_name in module_candidates(identifier, mod.modulename):
376            module: pdoc.doc.Module | None = context["all_modules"].get(module_name)
377            if not module:
378                continue
379
380            for _, qualname in sources:
381                doc = module.get(qualname)
382                # Check if they have an object with the same name,
383                # and verify that it's pointing to the right thing and is public.
384                if (
385                    doc
386                    and (target_object is doc.obj or target_object is None)
387                    and context["is_public"](doc).strip()
388                ):
389                    if shorten:
390                        if module == mod:
391                            url_text = qualname
392                        else:
393                            url_text = doc.fullname
394                        if plain_text.endswith("()"):
395                            url_text += "()"
396                    else:
397                        url_text = plain_text
398                    return f'<a href="{relative_link(context["module"].modulename, doc.modulename)}#{qualname}">{url_text}</a>'
399
400        # No matches found.
401        return text
402
403    return Markup(
404        re.sub(
405            r"""
406            # Part 1: foo.bar or foo.bar() (without backticks)
407            (?<![/=?#&\.])  # heuristic: not part of a URL
408            # First part of the identifier (e.g. "foo") - this is optional for relative references.
409            (?:
410                \b
411                (?!\d)[a-zA-Z0-9_]+
412                |
413                \.*  # We may also start with multiple dots.
414            )
415            # Rest of the identifier (e.g. ".bar" or "..bar")
416            (?:
417                # A single dot or a dot surrounded with pygments highlighting.
418                (?:\.|</span><span\ class="o">\.</span><span\ class="n">)
419                (?!\d)[a-zA-Z0-9_]+
420            )+
421            (?:\(\)|\b(?!\(\)))  # we either end on () or on a word boundary.
422            (?!</a>)  # not an existing link
423            (?![/#])  # heuristic: not part of a URL
424
425            | # Part 2: `foo` or `foo()`. `foo.bar` is already covered with part 1.
426            (?<=<code>)
427                 (?!\d)[a-zA-Z0-9_]+
428            (?:\(\))?
429            (?=</code>(?!</a>))
430            """,
431            linkify_repl,
432            code,
433            flags=re.VERBOSE,
434        )
435    )
436
437
438@pass_context
439def link(context: Context, spec: tuple[str, str], text: str | None = None) -> str:
440    """Create a link for a specific `(modulename, qualname)` tuple."""
441    mod: pdoc.doc.Module = context["module"]
442    modulename, qualname = spec
443
444    # Check if the object we are interested is also imported and re-exposed in the current namespace.
445    # https://github.com/mitmproxy/pdoc/issues/490: We need to do this for every level, not just the tail.
446    doc: pdoc.doc.Doc | None = mod
447    for part in qualname.split("."):
448        doc = doc.get(part) if isinstance(doc, pdoc.doc.Namespace) else None
449        if not (
450            doc
451            and doc.taken_from[0] == modulename
452            and context["is_public"](doc).strip()
453        ):
454            break
455    else:
456        # everything down to the tail is imported and re-exposed.
457        if text:
458            text = text.replace(f"{modulename}.", f"{mod.modulename}.")
459        modulename = mod.modulename
460
461    if mod.modulename == modulename:
462        fullname = qualname
463    else:
464        fullname = f"{modulename}.{qualname}".removesuffix(".")
465
466    if qualname:
467        qualname = f"#{qualname}"
468    if modulename in context["all_modules"]:
469        return Markup(
470            f'<a href="{relative_link(context["module"].modulename, modulename)}{qualname}">{text or fullname}</a>'
471        )
472    return text or fullname
473
474
475def edit_url(
476    modulename: str, is_package: bool, mapping: Mapping[str, str]
477) -> str | None:
478    """Create a link to edit a particular file in the used version control system."""
479    for m, prefix in mapping.items():
480        if m == modulename or modulename.startswith(f"{m}."):
481            filename = modulename[len(m) + 1 :].replace(".", "/")
482            if is_package:
483                filename = f"{filename}/__init__.py".lstrip("/")
484            else:
485                filename += ".py"
486            return f"{prefix}{filename}"
487    return None
488
489
490def root_module_name(all_modules: Mapping[str, pdoc.doc.Module]) -> str | None:
491    """
492    Return the name of the (unique) top-level module, or `None`
493    if no such module exists.
494
495    For example, assuming `foo`, `foo.bar`, and `foo.baz` are documented,
496    this function will return `foo`. If `foo` and `bar` are documented,
497    this function will return `None` as there is no unique top-level module.
498    """
499    shortest_name = min(all_modules, key=len, default=None)
500    prefix = f"{shortest_name}."
501    all_others_are_submodules = all(
502        x.startswith(prefix) or x == shortest_name for x in all_modules
503    )
504    if all_others_are_submodules:
505        return shortest_name
506    else:
507        return None
508
509
510def minify_css(css: str) -> str:
511    """Do some very basic CSS minification."""
512    css = re.sub(r"[ ]{4}|\n|(?<=[:{}]) | (?=[{}])", "", css)
513    css = re.sub(
514        r"/\*.+?\*/", lambda m: m.group(0) if m.group(0).startswith("/*!") else "", css
515    )
516    return Markup(css.replace("<style", "\n<style"))
517
518
519@contextmanager
520def defuse_unsafe_reprs():
521    """This decorator is applied by pdoc before calling an object's repr().
522    It applies some heuristics to patch our sensitive information.
523    For example, `os.environ`'s default `__repr__` implementation exposes all
524    local secrets.
525    """
526    with patch.object(os._Environ, "__repr__", lambda self: "os.environ"):
527        yield
528
529
530class DefaultMacroExtension(ext.Extension):
531    """
532    This extension provides a new `{% defaultmacro %}` statement, which defines a macro only if it does not exist.
533
534    For example,
535
536    ```html+jinja
537    {% defaultmacro example() %}
538        test 123
539    {% enddefaultmacro %}
540    ```
541
542    is equivalent to
543
544    ```html+jinja
545    {% macro default_example() %}
546    test 123
547    {% endmacro %}
548    {% if not example %}
549        {% macro example() %}
550            test 123
551        {% endmacro %}
552    {% endif %}
553    ```
554
555    Additionally, the default implementation is also available as `default_$macroname`, which makes it possible
556    to reference it in the override.
557    """
558
559    tags = {"defaultmacro"}
560
561    def parse(self, parser):
562        m = nodes.Macro(lineno=next(parser.stream).lineno)
563        name = parser.parse_assign_target(name_only=True).name
564        m.name = f"default_{name}"
565        parser.parse_signature(m)
566        m.body = parser.parse_statements(("name:enddefaultmacro",), drop_needle=True)
567
568        if_stmt = nodes.If(
569            nodes.Not(nodes.Name(name, "load")),
570            [nodes.Macro(name, m.args, m.defaults, m.body)],
571            [],
572            [],
573        )
574        return [m, if_stmt]