pdoc.render_helpers

  1from __future__ import annotations
  2
  3from collections.abc import Collection
  4from collections.abc import Iterable
  5from collections.abc import Mapping
  6from contextlib import contextmanager
  7import html
  8import inspect
  9import os
 10import re
 11from unittest.mock import patch
 12import warnings
 13
 14from jinja2 import ext
 15from jinja2 import nodes
 16import pygments.formatters
 17import pygments.lexers
 18
 19try:
 20    # Jinja2 >= 3.0
 21    from jinja2 import pass_context  # type: ignore
 22except ImportError:  # pragma: no cover
 23    from jinja2 import contextfilter as pass_context  # type: ignore
 24
 25from jinja2.runtime import Context
 26from markupsafe import Markup
 27
 28import pdoc.markdown2
 29
 30from . import docstrings
 31from ._compat import cache
 32from ._compat import removesuffix
 33
 34lexer = pygments.lexers.PythonLexer()
 35"""
 36The pygments lexer used for pdoc.render_helpers.highlight.
 37Overwrite this to configure pygments lexing.
 38"""
 39
 40formatter = pygments.formatters.HtmlFormatter(
 41    cssclass="pdoc-code codehilite",
 42    linenos="inline",
 43    anchorlinenos=True,
 44)
 45"""
 46The pygments formatter used for pdoc.render_helpers.highlight.
 47Overwrite this to configure pygments highlighting of code blocks.
 48
 49The usage of the `.codehilite` CSS selector in custom templates is deprecated since pdoc 10, use `.pdoc-code` instead.
 50"""
 51
 52signature_formatter = pygments.formatters.HtmlFormatter(nowrap=True)
 53"""
 54The pygments formatter used for pdoc.render_helpers.format_signature.
 55Overwrite this to configure pygments highlighting of signatures.
 56"""
 57
 58# Keep in sync with the documentation in pdoc/__init__.py!
 59markdown_extensions = {
 60    "alerts": None,
 61    "code-friendly": None,
 62    "cuddled-lists": None,
 63    "fenced-code-blocks": {"cssclass": formatter.cssclass},
 64    "footnotes": None,
 65    "header-ids": None,
 66    "link-patterns": None,
 67    "markdown-in-html": None,
 68    "mermaid": None,
 69    "pyshell": None,
 70    "strike": None,
 71    "tables": None,
 72    "task_list": None,
 73    "toc": {"depth": 2},
 74}
 75"""
 76The default extensions loaded for `markdown2`.
 77Overwrite this to configure Markdown rendering.
 78"""
 79markdown_link_patterns = [
 80    (
 81        re.compile(
 82            r"""
 83            \b
 84            (
 85                (?:https?://|(?<!//)www\.)    # prefix - https:// or www.
 86                \w[\w_\-]*(?:\.\w[\w_\-]*)*   # host
 87                [^<>\s"']*                    # rest of url
 88                (?<![?!.,:*_~);])             # exclude trailing punctuation
 89                (?=[?!.,:*_~);]?(?:[<\s]|$))  # make sure that we're not followed by " or ', i.e. we're outside of href="...".
 90            )
 91        """,
 92            re.X,
 93        ),
 94        r"\1",
 95    )
 96]
 97"""
 98Link pattern used for markdown2's [`link-patterns` extra](https://github.com/trentm/python-markdown2/wiki/link-patterns).
 99"""
100
101
102@cache
103def highlight(doc: pdoc.doc.Doc) -> str:
104    """Highlight the source code of a documentation object using pygments."""
105    if isinstance(doc, str):  # pragma: no cover
106        warnings.warn(
107            "Passing a string to the `highlight` render helper is deprecated, pass a pdoc.doc.Doc object instead.",
108            DeprecationWarning,
109        )
110        return Markup(pygments.highlight(doc, lexer, formatter))
111
112    # set up correct line numbers and anchors
113    formatter.linespans = doc.qualname or "L"
114    formatter.linenostart = doc.source_lines[0] + 1 if doc.source_lines else 1
115    return Markup(pygments.highlight(doc.source, lexer, formatter))
116
117
118def format_signature(sig: inspect.Signature, colon: bool) -> str:
119    """Format and highlight a function signature using pygments. Returns HTML."""
120    # First get a list with all params as strings.
121    result = pdoc.doc._PrettySignature._params(sig)  # type: ignore
122    return_annot = pdoc.doc._PrettySignature._return_annotation_str(sig)  # type: ignore
123
124    multiline = (
125        sum(len(x) + 2 for x in result) + len(return_annot)
126        > pdoc.doc._PrettySignature.MULTILINE_CUTOFF
127    )
128
129    def _try_highlight(code: str) -> str:
130        """Try to highlight a piece of code using pygments, but return the input as-is if pygments detects errors."""
131        pretty = pygments.highlight(code, lexer, signature_formatter).strip()
132        if '<span class="err">' not in pretty:
133            return pretty
134        else:
135            return html.escape(code)
136
137    # Next, individually highlight each parameter using pygments and wrap it in a span.param.
138    # This later allows us to properly control line breaks.
139    pretty_result = []
140    for i, param in enumerate(result):
141        pretty = _try_highlight(param)
142        if multiline:
143            pretty = f"""<span class="param">\t{pretty},</span>"""
144        else:
145            pretty = f"""<span class="param">{pretty}, </span>"""
146        pretty_result.append(pretty)
147
148    # remove last comma.
149    if pretty_result:
150        pretty_result[-1] = pretty_result[-1].rpartition(",")[0] + "</span>"
151
152    # Add return annotation.
153    anno = ")"
154    if return_annot:
155        anno += f" -> {_try_highlight(return_annot)}"
156    if colon:
157        anno += ":"
158    if return_annot or colon:
159        anno = f'<span class="return-annotation">{anno}</span>'
160
161    rendered = "(" + "".join(pretty_result) + anno
162
163    if multiline:
164        rendered = f'<span class="signature pdoc-code multiline">{rendered}</span>'
165    else:
166        rendered = f'<span class="signature pdoc-code condensed">{rendered}</span>'
167
168    return Markup(rendered)
169
170
171@cache
172def to_html(docstring: str) -> str:
173    """
174    Convert `docstring` from Markdown to HTML.
175    """
176    # careful: markdown2 returns a subclass of str with an extra
177    # .toc_html attribute. don't further process the result,
178    # otherwise this attribute will be lost.
179    return pdoc.markdown2.markdown(  # type: ignore
180        docstring,
181        extras=markdown_extensions,
182        link_patterns=markdown_link_patterns,
183    )
184
185
186@pass_context
187def to_markdown_with_context(context: Context, docstring: str) -> str:
188    """
189    Converts `docstring` from a custom docformat to Markdown (if necessary), and then from Markdown to HTML.
190    """
191    module: pdoc.doc.Module = context["module"]
192    docformat: str = context["docformat"]
193    return to_markdown(docstring, module, docformat)
194
195
196def to_markdown(docstring: str, module: pdoc.doc.Module, default_docformat: str) -> str:
197    docformat = getattr(module.obj, "__docformat__", default_docformat) or ""
198    return docstrings.convert(docstring, docformat, module.source_file)
199
200
201def possible_sources(
202    all_modules: Collection[str], identifier: str
203) -> Iterable[tuple[str, str]]:
204    """
205    For a given identifier, return all possible sources where it could originate from.
206    For example, assume `examplepkg._internal.Foo` with all_modules=["examplepkg"].
207    This could be a Foo class in _internal.py, or a nested `class _internal: class Foo` in examplepkg.
208    We return both candidates as we don't know if _internal.py exists.
209    It may not be in all_modules because it's been excluded by `__all__`.
210    However, if `examplepkg._internal` is in all_modules we know that it can only be that option.
211
212    >>> possible_sources(["examplepkg"], "examplepkg.Foo.bar")
213    examplepkg.Foo, bar
214    examplepkg, Foo.bar
215    """
216    if identifier in all_modules:
217        yield identifier, ""
218        return
219
220    modulename = identifier
221    qualname = None
222    while modulename:
223        modulename, _, add = modulename.rpartition(".")
224        qualname = f"{add}.{qualname}" if qualname else add
225        yield modulename, qualname
226        if modulename in all_modules:
227            return
228    raise ValueError(f"Invalid identifier: {identifier}")
229
230
231def split_identifier(all_modules: Collection[str], fullname: str) -> tuple[str, str]:
232    """
233    Split an identifier into a `(modulename, qualname)` tuple. For example, `pdoc.render_helpers.split_identifier`
234    would be split into `("pdoc.render_helpers","split_identifier")`. This is necessary to generate links to the
235    correct module.
236    """
237    warnings.warn(
238        "pdoc.render_helpers.split_identifier is deprecated and will be removed in a future release. "
239        "Use pdoc.render_helpers.possible_sources instead.",
240        DeprecationWarning,
241    )
242    *_, last = possible_sources(all_modules, fullname)
243    return last
244
245
246def _relative_link(current: list[str], target: list[str]) -> str:
247    if target == current:
248        return f"../{target[-1]}.html"
249    elif target[: len(current)] == current:
250        return "/".join(target[len(current) :]) + ".html"
251    else:
252        return "../" + _relative_link(current[:-1], target)
253
254
255@cache
256def relative_link(current_module: str, target_module: str) -> str:
257    """Compute the relative link to another module's HTML file."""
258    if current_module == target_module:
259        return ""
260    return _relative_link(
261        current_module.split(".")[:-1],
262        target_module.split("."),
263    )
264
265
266def qualname_candidates(identifier: str, context_qualname: str) -> list[str]:
267    """
268    Given an identifier in a current namespace, return all possible qualnames in the current module.
269    For example, if we are in Foo's subclass Bar and `baz()` is the identifier,
270    return `Foo.Bar.baz()`, `Foo.baz()`, and `baz()`.
271    """
272    end = len(context_qualname)
273    ret = []
274    while end > 0:
275        ret.append(f"{context_qualname[:end]}.{identifier}")
276        end = context_qualname.rfind(".", 0, end)
277    ret.append(identifier)
278    return ret
279
280
281def module_candidates(identifier: str, current_module: str) -> Iterable[str]:
282    """
283    Given an identifier and the current module name, return the module names we should look at
284    to find where the target object is exposed. Module names are ordered by preferences, i.e.
285    we always prefer the current module and then top-level modules over their children.
286
287    >>> module_candidates("foo.bar.baz", "qux")
288    qux
289    foo
290    foo.bar
291    foo.bar.baz
292    >>> module_candidates("foo.bar.baz", "foo.bar")
293    foo.bar
294    foo
295    foo.bar.baz
296    """
297    yield current_module
298
299    end = identifier.find(".")
300    while end > 0:
301        if (name := identifier[:end]) != current_module:
302            yield name
303        end = identifier.find(".", end + 1)
304
305    if identifier != current_module:
306        yield identifier
307
308
309@pass_context
310def linkify(
311    context: Context, code: str, namespace: str = "", shorten: bool = True
312) -> str:
313    """
314    Link all identifiers in a block of text. Identifiers referencing unknown modules or modules that
315    are not rendered at the moment will be ignored.
316    A piece of text is considered to be an identifier if it either contains a `.` or is surrounded by `<code>` tags.
317
318    If `shorten` is True, replace identifiers with short forms where possible.
319    For example, replace "current_module.Foo" with "Foo". This is useful for annotations
320    (which are verbose), but undesired for docstrings (where we want to preserve intent).
321    """
322
323    def linkify_repl(m: re.Match):
324        """
325        Resolve `text` to the most suitable documentation object.
326        """
327        text = m.group(0)
328        plain_text = text.replace(
329            '</span><span class="o">.</span><span class="n">', "."
330        )
331        identifier = removesuffix(plain_text, "()")
332        mod: pdoc.doc.Module = context["module"]
333
334        # Check if this is a relative reference. These cannot be local and need to be resolved.
335        if identifier.startswith("."):
336            taken_from_mod = mod
337            if namespace and (ns := mod.get(namespace)):
338                # Imported from somewhere else, so the relative reference should be from the original module.
339                taken_from_mod = context["all_modules"].get(ns.taken_from[0], mod)
340            if taken_from_mod.is_package:
341                # If we are in __init__.py, we want `.foo` to refer to a child module.
342                parent_module = taken_from_mod.modulename
343            else:
344                # If we are in a leaf module, we want `.foo` to refer to the adjacent module.
345                parent_module = taken_from_mod.modulename.rpartition(".")[0]
346            while identifier.startswith(".."):
347                identifier = identifier[1:]
348                parent_module = parent_module.rpartition(".")[0]
349            identifier = parent_module + identifier
350        else:
351            # Is this a local reference within this module?
352            for qualname in qualname_candidates(identifier, namespace):
353                doc = mod.get(qualname)
354                if doc and context["is_public"](doc).strip():
355                    return f'<a href="#{qualname}">{plain_text}</a>'
356
357        # Is this a reference pointing straight at a module?
358        if identifier in context["all_modules"]:
359            return f'<a href="{relative_link(context["module"].modulename, identifier)}">{identifier}</a>'
360
361        try:
362            sources = list(possible_sources(context["all_modules"], identifier))
363        except ValueError:
364            # possible_sources did not find a parent module.
365            return text
366
367        # Try to find the actual target object so that we can then later verify
368        # that objects exposed at a parent module with the same name point to it.
369        target_object = None
370        for module_name, qualname in sources:
371            if doc := context["all_modules"].get(module_name, {}).get(qualname):
372                target_object = doc.obj
373                break
374
375        # Look at the different modules where our target object may be exposed.
376        for module_name in module_candidates(identifier, mod.modulename):
377            module: pdoc.doc.Module | None = context["all_modules"].get(module_name)
378            if not module:
379                continue
380
381            for _, qualname in sources:
382                doc = module.get(qualname)
383                # Check if they have an object with the same name,
384                # and verify that it's pointing to the right thing and is public.
385                if (
386                    doc
387                    and (target_object is doc.obj or target_object is None)
388                    and context["is_public"](doc).strip()
389                ):
390                    if shorten:
391                        if module == mod:
392                            url_text = qualname
393                        else:
394                            url_text = doc.fullname
395                        if plain_text.endswith("()"):
396                            url_text += "()"
397                    else:
398                        url_text = plain_text
399                    return f'<a href="{relative_link(context["module"].modulename, doc.modulename)}#{qualname}">{url_text}</a>'
400
401        # No matches found.
402        return text
403
404    return Markup(
405        re.sub(
406            r"""
407            # Part 1: foo.bar or foo.bar() (without backticks)
408            (?<![/=?#&\.])  # heuristic: not part of a URL
409            # First part of the identifier (e.g. "foo") - this is optional for relative references.
410            (?:
411                \b
412                (?!\d)[a-zA-Z0-9_]+
413                |
414                \.*  # We may also start with multiple dots.
415            )
416            # Rest of the identifier (e.g. ".bar" or "..bar")
417            (?:
418                # A single dot or a dot surrounded with pygments highlighting.
419                (?:\.|</span><span\ class="o">\.</span><span\ class="n">)
420                (?!\d)[a-zA-Z0-9_]+
421            )+
422            (?:\(\)|\b(?!\(\)))  # we either end on () or on a word boundary.
423            (?!</a>)  # not an existing link
424            (?![/#])  # heuristic: not part of a URL
425
426            | # Part 2: `foo` or `foo()`. `foo.bar` is already covered with part 1.
427            (?<=<code>)
428                 (?!\d)[a-zA-Z0-9_]+
429            (?:\(\))?
430            (?=</code>(?!</a>))
431            """,
432            linkify_repl,
433            code,
434            flags=re.VERBOSE,
435        )
436    )
437
438
439@pass_context
440def link(context: Context, spec: tuple[str, str], text: str | None = None) -> str:
441    """Create a link for a specific `(modulename, qualname)` tuple."""
442    mod: pdoc.doc.Module = context["module"]
443    modulename, qualname = spec
444
445    # Check if the object we are interested is also imported and re-exposed in the current namespace.
446    # https://github.com/mitmproxy/pdoc/issues/490: We need to do this for every level, not just the tail.
447    doc: pdoc.doc.Doc | None = mod
448    for part in qualname.split("."):
449        doc = doc.get(part) if isinstance(doc, pdoc.doc.Namespace) else None
450        if not (
451            doc
452            and doc.taken_from[0] == modulename
453            and context["is_public"](doc).strip()
454        ):
455            break
456    else:
457        # everything down to the tail is imported and re-exposed.
458        if text:
459            text = text.replace(f"{modulename}.", f"{mod.modulename}.")
460        modulename = mod.modulename
461
462    if mod.modulename == modulename:
463        fullname = qualname
464    else:
465        fullname = removesuffix(f"{modulename}.{qualname}", ".")
466
467    if qualname:
468        qualname = f"#{qualname}"
469    if modulename in context["all_modules"]:
470        return Markup(
471            f'<a href="{relative_link(context["module"].modulename, modulename)}{qualname}">{text or fullname}</a>'
472        )
473    return text or fullname
474
475
476def edit_url(
477    modulename: str, is_package: bool, mapping: Mapping[str, str]
478) -> str | None:
479    """Create a link to edit a particular file in the used version control system."""
480    for m, prefix in mapping.items():
481        if m == modulename or modulename.startswith(f"{m}."):
482            filename = modulename[len(m) + 1 :].replace(".", "/")
483            if is_package:
484                filename = f"{filename}/__init__.py".lstrip("/")
485            else:
486                filename += ".py"
487            return f"{prefix}{filename}"
488    return None
489
490
491def root_module_name(all_modules: Mapping[str, pdoc.doc.Module]) -> str | None:
492    """
493    Return the name of the (unique) top-level module, or `None`
494    if no such module exists.
495
496    For example, assuming `foo`, `foo.bar`, and `foo.baz` are documented,
497    this function will return `foo`. If `foo` and `bar` are documented,
498    this function will return `None` as there is no unique top-level module.
499    """
500    shortest_name = min(all_modules, key=len, default=None)
501    prefix = f"{shortest_name}."
502    all_others_are_submodules = all(
503        x.startswith(prefix) or x == shortest_name for x in all_modules
504    )
505    if all_others_are_submodules:
506        return shortest_name
507    else:
508        return None
509
510
511def minify_css(css: str) -> str:
512    """Do some very basic CSS minification."""
513    css = re.sub(r"[ ]{4}|\n|(?<=[:{}]) | (?=[{}])", "", css)
514    css = re.sub(
515        r"/\*.+?\*/", lambda m: m.group(0) if m.group(0).startswith("/*!") else "", css
516    )
517    return Markup(css.replace("<style", "\n<style"))
518
519
520@contextmanager
521def defuse_unsafe_reprs():
522    """This decorator is applied by pdoc before calling an object's repr().
523    It applies some heuristics to patch our sensitive information.
524    For example, `os.environ`'s default `__repr__` implementation exposes all
525    local secrets.
526    """
527    with patch.object(os._Environ, "__repr__", lambda self: "os.environ"):
528        yield
529
530
531class DefaultMacroExtension(ext.Extension):
532    """
533    This extension provides a new `{% defaultmacro %}` statement, which defines a macro only if it does not exist.
534
535    For example,
536
537    ```html+jinja
538    {% defaultmacro example() %}
539        test 123
540    {% enddefaultmacro %}
541    ```
542
543    is equivalent to
544
545    ```html+jinja
546    {% macro default_example() %}
547    test 123
548    {% endmacro %}
549    {% if not example %}
550        {% macro example() %}
551            test 123
552        {% endmacro %}
553    {% endif %}
554    ```
555
556    Additionally, the default implementation is also available as `default_$macroname`, which makes it possible
557    to reference it in the override.
558    """
559
560    tags = {"defaultmacro"}
561
562    def parse(self, parser):
563        m = nodes.Macro(lineno=next(parser.stream).lineno)
564        name = parser.parse_assign_target(name_only=True).name
565        m.name = f"default_{name}"
566        parser.parse_signature(m)
567        m.body = parser.parse_statements(("name:enddefaultmacro",), drop_needle=True)
568
569        if_stmt = nodes.If(
570            nodes.Not(nodes.Name(name, "load")),
571            [nodes.Macro(name, m.args, m.defaults, m.body)],
572            [],
573            [],
574        )
575        return [m, if_stmt]