pdoc.render_helpers

  1from __future__ import annotations
  2
  3from collections.abc import Collection
  4from collections.abc import Iterable
  5from collections.abc import Mapping
  6from contextlib import contextmanager
  7import html
  8import inspect
  9import os
 10import re
 11from unittest.mock import patch
 12import warnings
 13
 14from jinja2 import ext
 15from jinja2 import nodes
 16import pygments.formatters
 17import pygments.lexers
 18
 19try:
 20    # Jinja2 >= 3.0
 21    from jinja2 import pass_context  # type: ignore
 22except ImportError:  # pragma: no cover
 23    from jinja2 import contextfilter as pass_context  # type: ignore
 24
 25from jinja2.runtime import Context
 26from markupsafe import Markup
 27
 28import pdoc.markdown2
 29
 30from . import docstrings
 31from ._compat import cache
 32from ._compat import removesuffix
 33
 34lexer = pygments.lexers.PythonLexer()
 35"""
 36The pygments lexer used for pdoc.render_helpers.highlight.
 37Overwrite this to configure pygments lexing.
 38"""
 39
 40formatter = pygments.formatters.HtmlFormatter(
 41    cssclass="pdoc-code codehilite",
 42    linenos="inline",
 43    anchorlinenos=True,
 44)
 45"""
 46The pygments formatter used for pdoc.render_helpers.highlight.
 47Overwrite this to configure pygments highlighting of code blocks.
 48
 49The usage of the `.codehilite` CSS selector in custom templates is deprecated since pdoc 10, use `.pdoc-code` instead.
 50"""
 51
 52signature_formatter = pygments.formatters.HtmlFormatter(nowrap=True)
 53"""
 54The pygments formatter used for pdoc.render_helpers.format_signature.
 55Overwrite this to configure pygments highlighting of signatures.
 56"""
 57
 58# Keep in sync with the documentation in pdoc/__init__.py!
 59markdown_extensions = {
 60    "alerts": None,
 61    "code-friendly": None,
 62    "cuddled-lists": None,
 63    "fenced-code-blocks": {"cssclass": formatter.cssclass},
 64    "footnotes": None,
 65    "header-ids": None,
 66    "link-patterns": None,
 67    "markdown-in-html": None,
 68    "mermaid": None,
 69    "pyshell": None,
 70    "strike": None,
 71    "tables": None,
 72    "task_list": None,
 73    "toc": {"depth": 2},
 74}
 75"""
 76The default extensions loaded for `markdown2`.
 77Overwrite this to configure Markdown rendering.
 78"""
 79markdown_link_patterns = [
 80    (
 81        re.compile(
 82            r"""
 83            \b
 84            (
 85                (?:https?://|(?<!//)www\.)    # prefix - https:// or www.
 86                \w[\w_\-]*(?:\.\w[\w_\-]*)*   # host
 87                [^<>\s"']*                    # rest of url
 88                (?<![?!.,:*_~);])             # exclude trailing punctuation
 89                (?=[?!.,:*_~);]?(?:[<\s]|$))  # make sure that we're not followed by " or ', i.e. we're outside of href="...".
 90            )
 91        """,
 92            re.X,
 93        ),
 94        r"\1",
 95    )
 96]
 97"""
 98Link pattern used for markdown2's [`link-patterns` extra](https://github.com/trentm/python-markdown2/wiki/link-patterns).
 99"""
100
101
102@cache
103def highlight(doc: pdoc.doc.Doc) -> str:
104    """Highlight the source code of a documentation object using pygments."""
105    if isinstance(doc, str):  # pragma: no cover
106        warnings.warn(
107            "Passing a string to the `highlight` render helper is deprecated, pass a pdoc.doc.Doc object instead.",
108            DeprecationWarning,
109        )
110        return Markup(pygments.highlight(doc, lexer, formatter))
111
112    # set up correct line numbers and anchors
113    formatter.linespans = doc.qualname or "L"
114    formatter.linenostart = doc.source_lines[0] + 1 if doc.source_lines else 1
115    return Markup(pygments.highlight(doc.source, lexer, formatter))
116
117
118def format_signature(sig: inspect.Signature, colon: bool) -> str:
119    """Format and highlight a function signature using pygments. Returns HTML."""
120    # First get a list with all params as strings.
121    result = pdoc.doc._PrettySignature._params(sig)  # type: ignore
122    return_annot = pdoc.doc._PrettySignature._return_annotation_str(sig)  # type: ignore
123
124    multiline = (
125        sum(len(x) + 2 for x in result) + len(return_annot)
126        > pdoc.doc._PrettySignature.MULTILINE_CUTOFF
127    )
128
129    def _try_highlight(code: str) -> str:
130        """Try to highlight a piece of code using pygments, but return the input as-is if pygments detects errors."""
131        pretty = pygments.highlight(code, lexer, signature_formatter).strip()
132        if '<span class="err">' not in pretty:
133            return pretty
134        else:
135            return html.escape(code)
136
137    # Next, individually highlight each parameter using pygments and wrap it in a span.param.
138    # This later allows us to properly control line breaks.
139    pretty_result = []
140    for i, param in enumerate(result):
141        pretty = _try_highlight(param)
142        if multiline:
143            pretty = f"""<span class="param">\t{pretty},</span>"""
144        else:
145            pretty = f"""<span class="param">{pretty}, </span>"""
146        pretty_result.append(pretty)
147
148    # remove last comma.
149    if pretty_result:
150        pretty_result[-1] = pretty_result[-1].rpartition(",")[0] + "</span>"
151
152    # Add return annotation.
153    anno = ")"
154    if return_annot:
155        anno += f" -> {_try_highlight(return_annot)}"
156    if colon:
157        anno += ":"
158    if return_annot or colon:
159        anno = f'<span class="return-annotation">{anno}</span>'
160
161    rendered = "(" + "".join(pretty_result) + anno
162
163    if multiline:
164        rendered = f'<span class="signature pdoc-code multiline">{rendered}</span>'
165    else:
166        rendered = f'<span class="signature pdoc-code condensed">{rendered}</span>'
167
168    return Markup(rendered)
169
170
171@cache
172def to_html(docstring: str) -> str:
173    """
174    Convert `docstring` from Markdown to HTML.
175    """
176    # careful: markdown2 returns a subclass of str with an extra
177    # .toc_html attribute. don't further process the result,
178    # otherwise this attribute will be lost.
179    return pdoc.markdown2.markdown(  # type: ignore
180        docstring,
181        extras=markdown_extensions,
182        link_patterns=markdown_link_patterns,
183    )
184
185
186@pass_context
187def to_markdown_with_context(context: Context, docstring: str) -> str:
188    """
189    Converts `docstring` from a custom docformat to Markdown (if necessary), and then from Markdown to HTML.
190    """
191    module: pdoc.doc.Module = context["module"]
192    docformat: str = context["docformat"]
193    return to_markdown(docstring, module, docformat)
194
195
196def to_markdown(docstring: str, module: pdoc.doc.Module, default_docformat: str) -> str:
197    docformat = getattr(module.obj, "__docformat__", default_docformat) or ""
198    return docstrings.convert(docstring, docformat, module.source_file)
199
200
201def possible_sources(
202    all_modules: Collection[str], identifier: str
203) -> Iterable[tuple[str, str]]:
204    """
205    For a given identifier, return all possible sources where it could originate from.
206    For example, assume `examplepkg._internal.Foo` with all_modules=["examplepkg"].
207    This could be a Foo class in _internal.py, or a nested `class _internal: class Foo` in examplepkg.
208    We return both candidates as we don't know if _internal.py exists.
209    It may not be in all_modules because it's been excluded by `__all__`.
210    However, if `examplepkg._internal` is in all_modules we know that it can only be that option.
211
212    >>> possible_sources(["examplepkg"], "examplepkg.Foo.bar")
213    examplepkg.Foo, bar
214    examplepkg, Foo.bar
215    """
216    if identifier in all_modules:
217        yield identifier, ""
218        return
219
220    modulename = identifier
221    qualname = None
222    while modulename:
223        modulename, _, add = modulename.rpartition(".")
224        qualname = f"{add}.{qualname}" if qualname else add
225        yield modulename, qualname
226        if modulename in all_modules:
227            return
228    raise ValueError(f"Invalid identifier: {identifier}")
229
230
231def split_identifier(all_modules: Collection[str], fullname: str) -> tuple[str, str]:
232    """
233    Split an identifier into a `(modulename, qualname)` tuple. For example, `pdoc.render_helpers.split_identifier`
234    would be split into `("pdoc.render_helpers","split_identifier")`. This is necessary to generate links to the
235    correct module.
236    """
237    warnings.warn(
238        "pdoc.render_helpers.split_identifier is deprecated and will be removed in a future release. "
239        "Use pdoc.render_helpers.possible_sources instead.",
240        DeprecationWarning,
241    )
242    *_, last = possible_sources(all_modules, fullname)
243    return last
244
245
246def _relative_link(current: list[str], target: list[str]) -> str:
247    if target == current:
248        return f"../{target[-1]}.html"
249    elif target[: len(current)] == current:
250        return "/".join(target[len(current) :]) + ".html"
251    else:
252        return "../" + _relative_link(current[:-1], target)
253
254
255@cache
256def relative_link(current_module: str, target_module: str) -> str:
257    """Compute the relative link to another module's HTML file."""
258    if current_module == target_module:
259        return ""
260    return _relative_link(
261        current_module.split(".")[:-1],
262        target_module.split("."),
263    )
264
265
266def qualname_candidates(identifier: str, context_qualname: str) -> list[str]:
267    """
268    Given an identifier in a current namespace, return all possible qualnames in the current module.
269    For example, if we are in Foo's subclass Bar and `baz()` is the identifier,
270    return `Foo.Bar.baz()`, `Foo.baz()`, and `baz()`.
271    """
272    end = len(context_qualname)
273    ret = []
274    while end > 0:
275        ret.append(f"{context_qualname[:end]}.{identifier}")
276        end = context_qualname.rfind(".", 0, end)
277    ret.append(identifier)
278    return ret
279
280
281def module_candidates(identifier: str, current_module: str) -> Iterable[str]:
282    """
283    Given an identifier and the current module name, return the module names we should look at
284    to find where the target object is exposed. Module names are ordered by preferences, i.e.
285    we always prefer the current module and then top-level modules over their children.
286
287    >>> module_candidates("foo.bar.baz", "qux")
288    qux
289    foo
290    foo.bar
291    foo.bar.baz
292    >>> module_candidates("foo.bar.baz", "foo.bar")
293    foo.bar
294    foo
295    foo.bar.baz
296    """
297    yield current_module
298
299    end = identifier.find(".")
300    while end > 0:
301        if (name := identifier[:end]) != current_module:
302            yield name
303        end = identifier.find(".", end + 1)
304
305    if identifier != current_module:
306        yield identifier
307
308
309@pass_context
310def linkify(context: Context, code: str, namespace: str = "") -> str:
311    """
312    Link all identifiers in a block of text. Identifiers referencing unknown modules or modules that
313    are not rendered at the moment will be ignored.
314    A piece of text is considered to be an identifier if it either contains a `.` or is surrounded by `<code>` tags.
315    """
316
317    def linkify_repl(m: re.Match):
318        """
319        Resolve `text` to the most suitable documentation object.
320        """
321        text = m.group(0)
322        plain_text = text.replace(
323            '</span><span class="o">.</span><span class="n">', "."
324        )
325        identifier = removesuffix(plain_text, "()")
326        mod: pdoc.doc.Module = context["module"]
327
328        # Check if this is a relative reference. These cannot be local and need to be resolved.
329        if identifier.startswith("."):
330            taken_from_mod = mod
331            if namespace and (ns := mod.get(namespace)):
332                # Imported from somewhere else, so the relative reference should be from the original module.
333                taken_from_mod = context["all_modules"].get(ns.taken_from[0], mod)
334            if taken_from_mod.is_package:
335                # If we are in __init__.py, we want `.foo` to refer to a child module.
336                parent_module = taken_from_mod.modulename
337            else:
338                # If we are in a leaf module, we want `.foo` to refer to the adjacent module.
339                parent_module = taken_from_mod.modulename.rpartition(".")[0]
340            while identifier.startswith(".."):
341                identifier = identifier[1:]
342                parent_module = parent_module.rpartition(".")[0]
343            identifier = parent_module + identifier
344        else:
345            # Is this a local reference within this module?
346            for qualname in qualname_candidates(identifier, namespace):
347                doc = mod.get(qualname)
348                if doc and context["is_public"](doc).strip():
349                    return f'<a href="#{qualname}">{plain_text}</a>'
350
351        # Is this a reference pointing straight at a module?
352        if identifier in context["all_modules"]:
353            return f'<a href="{relative_link(context["module"].modulename, identifier)}">{identifier}</a>'
354
355        try:
356            sources = list(possible_sources(context["all_modules"], identifier))
357        except ValueError:
358            # possible_sources did not find a parent module.
359            return text
360
361        # Try to find the actual target object so that we can then later verify
362        # that objects exposed at a parent module with the same name point to it.
363        target_object = None
364        for module_name, qualname in sources:
365            if doc := context["all_modules"].get(module_name, {}).get(qualname):
366                target_object = doc.obj
367                break
368
369        # Look at the different modules where our target object may be exposed.
370        for module_name in module_candidates(identifier, mod.modulename):
371            module: pdoc.doc.Module | None = context["all_modules"].get(module_name)
372            if not module:
373                continue
374
375            for _, qualname in sources:
376                doc = module.get(qualname)
377                # Check if they have an object with the same name,
378                # and verify that it's pointing to the right thing and is public.
379                if (
380                    doc
381                    and (target_object is doc.obj or target_object is None)
382                    and context["is_public"](doc).strip()
383                ):
384                    if module == mod:
385                        url_text = qualname
386                    else:
387                        url_text = doc.fullname
388                    if plain_text.endswith("()"):
389                        url_text += "()"
390                    return f'<a href="{relative_link(context["module"].modulename, doc.modulename)}#{qualname}">{url_text}</a>'
391
392        # No matches found.
393        return text
394
395    return Markup(
396        re.sub(
397            r"""
398            # Part 1: foo.bar or foo.bar() (without backticks)
399            (?<![/=?#&\.])  # heuristic: not part of a URL
400            # First part of the identifier (e.g. "foo") - this is optional for relative references.
401            (?:
402                \b
403                (?!\d)[a-zA-Z0-9_]+
404                |
405                \.*  # We may also start with multiple dots.
406            )
407            # Rest of the identifier (e.g. ".bar" or "..bar")
408            (?:
409                # A single dot or a dot surrounded with pygments highlighting.
410                (?:\.|</span><span\ class="o">\.</span><span\ class="n">)
411                (?!\d)[a-zA-Z0-9_]+
412            )+
413            (?:\(\)|\b(?!\(\)))  # we either end on () or on a word boundary.
414            (?!</a>)  # not an existing link
415            (?![/#])  # heuristic: not part of a URL
416
417            | # Part 2: `foo` or `foo()`. `foo.bar` is already covered with part 1.
418            (?<=<code>)
419                 (?!\d)[a-zA-Z0-9_]+
420            (?:\(\))?
421            (?=</code>(?!</a>))
422            """,
423            linkify_repl,
424            code,
425            flags=re.VERBOSE,
426        )
427    )
428
429
430@pass_context
431def link(context: Context, spec: tuple[str, str], text: str | None = None) -> str:
432    """Create a link for a specific `(modulename, qualname)` tuple."""
433    mod: pdoc.doc.Module = context["module"]
434    modulename, qualname = spec
435
436    # Check if the object we are interested is also imported and re-exposed in the current namespace.
437    # https://github.com/mitmproxy/pdoc/issues/490: We need to do this for every level, not just the tail.
438    doc: pdoc.doc.Doc | None = mod
439    for part in qualname.split("."):
440        doc = doc.get(part) if isinstance(doc, pdoc.doc.Namespace) else None
441        if not (
442            doc
443            and doc.taken_from[0] == modulename
444            and context["is_public"](doc).strip()
445        ):
446            break
447    else:
448        # everything down to the tail is imported and re-exposed.
449        if text:
450            text = text.replace(f"{modulename}.", f"{mod.modulename}.")
451        modulename = mod.modulename
452
453    if mod.modulename == modulename:
454        fullname = qualname
455    else:
456        fullname = removesuffix(f"{modulename}.{qualname}", ".")
457
458    if qualname:
459        qualname = f"#{qualname}"
460    if modulename in context["all_modules"]:
461        return Markup(
462            f'<a href="{relative_link(context["module"].modulename, modulename)}{qualname}">{text or fullname}</a>'
463        )
464    return text or fullname
465
466
467def edit_url(
468    modulename: str, is_package: bool, mapping: Mapping[str, str]
469) -> str | None:
470    """Create a link to edit a particular file in the used version control system."""
471    for m, prefix in mapping.items():
472        if m == modulename or modulename.startswith(f"{m}."):
473            filename = modulename[len(m) + 1 :].replace(".", "/")
474            if is_package:
475                filename = f"{filename}/__init__.py".lstrip("/")
476            else:
477                filename += ".py"
478            return f"{prefix}{filename}"
479    return None
480
481
482def root_module_name(all_modules: Mapping[str, pdoc.doc.Module]) -> str | None:
483    """
484    Return the name of the (unique) top-level module, or `None`
485    if no such module exists.
486
487    For example, assuming `foo`, `foo.bar`, and `foo.baz` are documented,
488    this function will return `foo`. If `foo` and `bar` are documented,
489    this function will return `None` as there is no unique top-level module.
490    """
491    shortest_name = min(all_modules, key=len, default=None)
492    prefix = f"{shortest_name}."
493    all_others_are_submodules = all(
494        x.startswith(prefix) or x == shortest_name for x in all_modules
495    )
496    if all_others_are_submodules:
497        return shortest_name
498    else:
499        return None
500
501
502def minify_css(css: str) -> str:
503    """Do some very basic CSS minification."""
504    css = re.sub(r"[ ]{4}|\n|(?<=[:{}]) | (?=[{}])", "", css)
505    css = re.sub(
506        r"/\*.+?\*/", lambda m: m.group(0) if m.group(0).startswith("/*!") else "", css
507    )
508    return Markup(css.replace("<style", "\n<style"))
509
510
511@contextmanager
512def defuse_unsafe_reprs():
513    """This decorator is applied by pdoc before calling an object's repr().
514    It applies some heuristics to patch our sensitive information.
515    For example, `os.environ`'s default `__repr__` implementation exposes all
516    local secrets.
517    """
518    with patch.object(os._Environ, "__repr__", lambda self: "os.environ"):
519        yield
520
521
522class DefaultMacroExtension(ext.Extension):
523    """
524    This extension provides a new `{% defaultmacro %}` statement, which defines a macro only if it does not exist.
525
526    For example,
527
528    ```html+jinja
529    {% defaultmacro example() %}
530        test 123
531    {% enddefaultmacro %}
532    ```
533
534    is equivalent to
535
536    ```html+jinja
537    {% macro default_example() %}
538    test 123
539    {% endmacro %}
540    {% if not example %}
541        {% macro example() %}
542            test 123
543        {% endmacro %}
544    {% endif %}
545    ```
546
547    Additionally, the default implementation is also available as `default_$macroname`, which makes it possible
548    to reference it in the override.
549    """
550
551    tags = {"defaultmacro"}
552
553    def parse(self, parser):
554        m = nodes.Macro(lineno=next(parser.stream).lineno)
555        name = parser.parse_assign_target(name_only=True).name
556        m.name = f"default_{name}"
557        parser.parse_signature(m)
558        m.body = parser.parse_statements(("name:enddefaultmacro",), drop_needle=True)
559
560        if_stmt = nodes.If(
561            nodes.Not(nodes.Name(name, "load")),
562            [nodes.Macro(name, m.args, m.defaults, m.body)],
563            [],
564            [],
565        )
566        return [m, if_stmt]