pdoc.render_helpers

  1from __future__ import annotations
  2
  3import html
  4import inspect
  5import os
  6import re
  7import warnings
  8from collections.abc import Collection
  9from collections.abc import Iterable
 10from collections.abc import Mapping
 11from contextlib import contextmanager
 12from unittest.mock import patch
 13
 14import pygments.formatters
 15import pygments.lexers
 16from jinja2 import ext
 17from jinja2 import nodes
 18
 19try:
 20    # Jinja2 >= 3.0
 21    from jinja2 import pass_context  # type: ignore
 22except ImportError:  # pragma: no cover
 23    from jinja2 import contextfilter as pass_context  # type: ignore
 24
 25from jinja2.runtime import Context
 26from markupsafe import Markup
 27
 28import pdoc.markdown2
 29
 30from . import docstrings
 31from ._compat import cache, removesuffix
 32
 33lexer = pygments.lexers.PythonLexer()
 34"""
 35The pygments lexer used for pdoc.render_helpers.highlight.
 36Overwrite this to configure pygments lexing.
 37"""
 38
 39formatter = pygments.formatters.HtmlFormatter(
 40    cssclass="pdoc-code codehilite",
 41    linenos="inline",
 42    anchorlinenos=True,
 43)
 44"""
 45The pygments formatter used for pdoc.render_helpers.highlight. 
 46Overwrite this to configure pygments highlighting of code blocks.
 47
 48The usage of the `.codehilite` CSS selector in custom templates is deprecated since pdoc 10, use `.pdoc-code` instead.
 49"""
 50
 51signature_formatter = pygments.formatters.HtmlFormatter(nowrap=True)
 52"""
 53The pygments formatter used for pdoc.render_helpers.format_signature. 
 54Overwrite this to configure pygments highlighting of signatures.
 55"""
 56
 57# Keep in sync with the documentation in pdoc/__init__.py!
 58markdown_extensions = {
 59    "code-friendly": None,
 60    "cuddled-lists": None,
 61    "fenced-code-blocks": {"cssclass": formatter.cssclass},
 62    "footnotes": None,
 63    "header-ids": None,
 64    "link-patterns": None,
 65    "markdown-in-html": None,
 66    "pyshell": None,
 67    "strike": None,
 68    "tables": None,
 69    "task_list": None,
 70    "toc": {"depth": 2},
 71}
 72"""
 73The default extensions loaded for `markdown2`.
 74Overwrite this to configure Markdown rendering.
 75"""
 76markdown_link_patterns = [
 77    (
 78        re.compile(
 79            r"""
 80            \b
 81            (
 82                (?:https?://|(?<!//)www\.)    # prefix - https:// or www.
 83                \w[\w_\-]*(?:\.\w[\w_\-]*)*   # host
 84                [^<>\s"']*                    # rest of url
 85                (?<![?!.,:*_~);])             # exclude trailing punctuation
 86                (?=[?!.,:*_~);]?(?:[<\s]|$))  # make sure that we're not followed by " or ', i.e. we're outside of href="...".
 87            )
 88        """,
 89            re.X,
 90        ),
 91        r"\1",
 92    )
 93]
 94"""
 95Link pattern used for markdown2's [`link-patterns` extra](https://github.com/trentm/python-markdown2/wiki/link-patterns).
 96"""
 97
 98
 99@cache
100def highlight(doc: pdoc.doc.Doc) -> str:
101    """Highlight the source code of a documentation object using pygments."""
102    if isinstance(doc, str):  # pragma: no cover
103        warnings.warn(
104            "Passing a string to the `highlight` render helper is deprecated, pass a pdoc.doc.Doc object instead.",
105            DeprecationWarning,
106        )
107        return Markup(pygments.highlight(doc, lexer, formatter))
108
109    # set up correct line numbers and anchors
110    formatter.linespans = doc.qualname or "L"
111    formatter.linenostart = doc.source_lines[0] + 1 if doc.source_lines else 1
112    return Markup(pygments.highlight(doc.source, lexer, formatter))
113
114
115def format_signature(sig: inspect.Signature, colon: bool) -> str:
116    """Format and highlight a function signature using pygments. Returns HTML."""
117    # First get a list with all params as strings.
118    result = pdoc.doc._PrettySignature._params(sig)  # type: ignore
119    return_annot = pdoc.doc._PrettySignature._return_annotation_str(sig)  # type: ignore
120
121    multiline = (
122        sum(len(x) + 2 for x in result) + len(return_annot)
123        > pdoc.doc._PrettySignature.MULTILINE_CUTOFF
124    )
125
126    def _try_highlight(code: str) -> str:
127        """Try to highlight a piece of code using pygments, but return the input as-is if pygments detects errors."""
128        pretty = pygments.highlight(code, lexer, signature_formatter).strip()
129        if '<span class="err">' not in pretty:
130            return pretty
131        else:
132            return html.escape(code)
133
134    # Next, individually highlight each parameter using pygments and wrap it in a span.param.
135    # This later allows us to properly control line breaks.
136    pretty_result = []
137    for i, param in enumerate(result):
138        pretty = _try_highlight(param)
139        if multiline:
140            pretty = f"""<span class="param">\t{pretty},</span>"""
141        else:
142            pretty = f"""<span class="param">{pretty}, </span>"""
143        pretty_result.append(pretty)
144
145    # remove last comma.
146    if pretty_result:
147        pretty_result[-1] = pretty_result[-1].rpartition(",")[0] + "</span>"
148
149    # Add return annotation.
150    anno = ")"
151    if return_annot:
152        anno += f" -> {_try_highlight(return_annot)}"
153    if colon:
154        anno += ":"
155    if return_annot or colon:
156        anno = f'<span class="return-annotation">{anno}</span>'
157
158    rendered = "(" + "".join(pretty_result) + anno
159
160    if multiline:
161        rendered = f'<span class="signature pdoc-code multiline">{rendered}</span>'
162    else:
163        rendered = f'<span class="signature pdoc-code condensed">{rendered}</span>'
164
165    return Markup(rendered)
166
167
168@cache
169def to_html(docstring: str) -> str:
170    """
171    Convert `docstring` from Markdown to HTML.
172    """
173    # careful: markdown2 returns a subclass of str with an extra
174    # .toc_html attribute. don't further process the result,
175    # otherwise this attribute will be lost.
176    return pdoc.markdown2.markdown(  # type: ignore
177        docstring,
178        extras=markdown_extensions,
179        link_patterns=markdown_link_patterns,
180    )
181
182
183@pass_context
184def to_markdown_with_context(context: Context, docstring: str) -> str:
185    """
186    Converts `docstring` from a custom docformat to Markdown (if necessary), and then from Markdown to HTML.
187    """
188    module: pdoc.doc.Module = context["module"]
189    docformat: str = context["docformat"]
190    return to_markdown(docstring, module, docformat)
191
192
193def to_markdown(docstring: str, module: pdoc.doc.Module, default_docformat: str) -> str:
194    docformat = getattr(module.obj, "__docformat__", default_docformat) or ""
195    return docstrings.convert(docstring, docformat, module.source_file)
196
197
198def possible_sources(
199    all_modules: Collection[str], identifier: str
200) -> Iterable[tuple[str, str]]:
201    """
202    For a given identifier, return all possible sources where it could originate from.
203    For example, assume `examplepkg._internal.Foo` with all_modules=["examplepkg"].
204    This could be a Foo class in _internal.py, or a nested `class _internal: class Foo` in examplepkg.
205    We return both candidates as we don't know if _internal.py exists.
206    It may not be in all_modules because it's been excluded by `__all__`.
207    However, if `examplepkg._internal` is in all_modules we know that it can only be that option.
208    """
209    if identifier in all_modules:
210        yield identifier, ""
211        return
212
213    modulename = identifier
214    qualname = None
215    while modulename:
216        modulename, _, add = modulename.rpartition(".")
217        qualname = f"{add}.{qualname}" if qualname else add
218        yield modulename, qualname
219        if modulename in all_modules:
220            return
221    raise ValueError(f"Invalid identifier: {identifier}")
222
223
224def split_identifier(all_modules: Collection[str], fullname: str) -> tuple[str, str]:
225    """
226    Split an identifier into a `(modulename, qualname)` tuple. For example, `pdoc.render_helpers.split_identifier`
227    would be split into `("pdoc.render_helpers","split_identifier")`. This is necessary to generate links to the
228    correct module.
229    """
230    warnings.warn(
231        "pdoc.render_helpers.split_identifier is deprecated and will be removed in a future release. "
232        "Use pdoc.render_helpers.possible_sources instead.",
233        DeprecationWarning,
234    )
235    *_, last = possible_sources(all_modules, fullname)
236    return last
237
238
239def _relative_link(current: list[str], target: list[str]) -> str:
240    if target == current:
241        return f"../{target[-1]}.html"
242    elif target[: len(current)] == current:
243        return "/".join(target[len(current) :]) + ".html"
244    else:
245        return "../" + _relative_link(current[:-1], target)
246
247
248@cache
249def relative_link(current_module: str, target_module: str) -> str:
250    """Compute the relative link to another module's HTML file."""
251    if current_module == target_module:
252        return ""
253    return _relative_link(
254        current_module.split(".")[:-1],
255        target_module.split("."),
256    )
257
258
259def qualname_candidates(identifier: str, context_qualname: str) -> list[str]:
260    """
261    Given an identifier in a current namespace, return all possible qualnames in the current module.
262    For example, if we are in Foo's subclass Bar and `baz()` is the identifier,
263    return `Foo.Bar.baz()`, `Foo.baz()`, and `baz()`.
264    """
265    end = len(context_qualname)
266    ret = []
267    while end > 0:
268        ret.append(f"{context_qualname[:end]}.{identifier}")
269        end = context_qualname.rfind(".", 0, end)
270    ret.append(identifier)
271    return ret
272
273
274@pass_context
275def linkify(context: Context, code: str, namespace: str = "") -> str:
276    """
277    Link all identifiers in a block of text. Identifiers referencing unknown modules or modules that
278    are not rendered at the moment will be ignored.
279    A piece of text is considered to be an identifier if it either contains a `.` or is surrounded by `<code>` tags.
280    """
281
282    def linkify_repl(m: re.Match):
283        text = m.group(0)
284        plain_text = text.replace(
285            '</span><span class="o">.</span><span class="n">', "."
286        )
287        identifier = removesuffix(plain_text, "()")
288
289        # Check if this is a local reference within this module?
290        mod: pdoc.doc.Module = context["module"]
291        for qualname in qualname_candidates(identifier, namespace):
292            doc = mod.get(qualname)
293            if doc and context["is_public"](doc).strip():
294                return f'<a href="#{qualname}">{plain_text}</a>'
295
296        module = ""
297        qualname = ""
298        try:
299            # Check if the object we are interested in is imported and re-exposed in the current namespace.
300            for module, qualname in possible_sources(
301                context["all_modules"], identifier
302            ):
303                doc = mod.get(qualname)
304                if (
305                    doc
306                    and doc.taken_from == (module, qualname)
307                    and context["is_public"](doc).strip()
308                ):
309                    if plain_text.endswith("()"):
310                        plain_text = f"{doc.fullname}()"
311                    else:
312                        plain_text = doc.fullname
313                    return f'<a href="#{qualname}">{plain_text}</a>'
314        except ValueError:
315            # possible_sources did not find a parent module.
316            return text
317        else:
318            # It's not, but we now know the parent module. Does the target exist?
319            doc = context["all_modules"][module]
320            if qualname:
321                assert isinstance(doc, pdoc.doc.Module)
322                doc = doc.get(qualname)
323            target_exists_and_public = (
324                doc is not None and context["is_public"](doc).strip()
325            )
326            if target_exists_and_public:
327                if qualname:
328                    qualname = f"#{qualname}"
329                return f'<a href="{relative_link(context["module"].modulename, module)}{qualname}">{plain_text}</a>'
330            else:
331                return text
332
333    return Markup(
334        re.sub(
335            r"""
336            # Part 1: foo.bar or foo.bar() (without backticks)
337            (?<![/=?#&])  # heuristic: not part of a URL
338            \b
339            
340            # First part of the identifier (e.g. "foo")    
341            (?!\d)[a-zA-Z0-9_]+
342            # Rest of the identifier (e.g. ".bar")
343            (?:
344                # A single dot or a dot surrounded with pygments highlighting.
345                (?:\.|</span><span\ class="o">\.</span><span\ class="n">)
346                (?!\d)[a-zA-Z0-9_]+
347            )+
348            (?:\(\)|\b(?!\(\)))  # we either end on () or on a word boundary.
349            (?!</a>)  # not an existing link
350            (?![/#])  # heuristic: not part of a URL
351
352            | # Part 2: `foo` or `foo()`. `foo.bar` is already covered with part 1.
353            (?<=<code>)
354                 (?!\d)[a-zA-Z0-9_]+
355            (?:\(\))?
356            (?=</code>(?!</a>))
357            """,
358            linkify_repl,
359            code,
360            flags=re.VERBOSE,
361        )
362    )
363
364
365@pass_context
366def link(context: Context, spec: tuple[str, str], text: str | None = None) -> str:
367    """Create a link for a specific `(modulename, qualname)` tuple."""
368    mod: pdoc.doc.Module = context["module"]
369    modulename, qualname = spec
370
371    # Check if the object we are interested is also imported and re-exposed in the current namespace.
372    # https://github.com/mitmproxy/pdoc/issues/490: We need to do this for every level, not just the tail.
373    doc: pdoc.doc.Doc | None = mod
374    for part in qualname.split("."):
375        doc = doc.get(part) if isinstance(doc, pdoc.doc.Namespace) else None
376        if not (
377            doc
378            and doc.taken_from[0] == modulename
379            and context["is_public"](doc).strip()
380        ):
381            break
382    else:
383        # everything down to the tail is imported and re-exposed.
384        if text:
385            text = text.replace(f"{modulename}.", f"{mod.modulename}.")
386        modulename = mod.modulename
387
388    if mod.modulename == modulename:
389        fullname = qualname
390    else:
391        fullname = removesuffix(f"{modulename}.{qualname}", ".")
392
393    if qualname:
394        qualname = f"#{qualname}"
395    if modulename in context["all_modules"]:
396        return Markup(
397            f'<a href="{relative_link(context["module"].modulename, modulename)}{qualname}">{text or fullname}</a>'
398        )
399    return text or fullname
400
401
402def edit_url(
403    modulename: str, is_package: bool, mapping: Mapping[str, str]
404) -> str | None:
405    """Create a link to edit a particular file in the used version control system."""
406    for m, prefix in mapping.items():
407        if m == modulename or modulename.startswith(f"{m}."):
408            filename = modulename[len(m) + 1 :].replace(".", "/")
409            if is_package:
410                filename = f"{filename}/__init__.py".lstrip("/")
411            else:
412                filename += ".py"
413            return f"{prefix}{filename}"
414    return None
415
416
417def root_module_name(all_modules: Mapping[str, pdoc.doc.Module]) -> str | None:
418    """
419    Return the name of the (unique) top-level module, or `None`
420    if no such module exists.
421
422    For example, assuming `foo`, `foo.bar`, and `foo.baz` are documented,
423    this function will return `foo`. If `foo` and `bar` are documented,
424    this function will return `None` as there is no unique top-level module.
425    """
426    shortest_name = min(all_modules, key=len, default=None)
427    prefix = f"{shortest_name}."
428    all_others_are_submodules = all(
429        x.startswith(prefix) or x == shortest_name for x in all_modules
430    )
431    if all_others_are_submodules:
432        return shortest_name
433    else:
434        return None
435
436
437def minify_css(css: str) -> str:
438    """Do some very basic CSS minification."""
439    css = re.sub(r"[ ]{4}|\n|(?<=[:{}]) | (?=[{}])", "", css)
440    css = re.sub(
441        r"/\*.+?\*/", lambda m: m.group(0) if m.group(0).startswith("/*!") else "", css
442    )
443    return Markup(css.replace("<style", "\n<style"))
444
445
446@contextmanager
447def defuse_unsafe_reprs():
448    """This decorator is applied by pdoc before calling an object's repr().
449    It applies some heuristics to patch our sensitive information.
450    For example, `os.environ`'s default `__repr__` implementation exposes all
451    local secrets.
452    """
453    with patch.object(os._Environ, "__repr__", lambda self: "os.environ"):
454        yield
455
456
457class DefaultMacroExtension(ext.Extension):
458    """
459    This extension provides a new `{% defaultmacro %}` statement, which defines a macro only if it does not exist.
460
461    For example,
462
463    ```html+jinja
464    {% defaultmacro example() %}
465        test 123
466    {% enddefaultmacro %}
467    ```
468
469    is equivalent to
470
471    ```html+jinja
472    {% macro default_example() %}
473    test 123
474    {% endmacro %}
475    {% if not example %}
476        {% macro example() %}
477            test 123
478        {% endmacro %}
479    {% endif %}
480    ```
481
482    Additionally, the default implementation is also available as `default_$macroname`, which makes it possible
483    to reference it in the override.
484    """
485
486    tags = {"defaultmacro"}
487
488    def parse(self, parser):
489        m = nodes.Macro(lineno=next(parser.stream).lineno)
490        name = parser.parse_assign_target(name_only=True).name
491        m.name = f"default_{name}"
492        parser.parse_signature(m)
493        m.body = parser.parse_statements(("name:enddefaultmacro",), drop_needle=True)
494
495        if_stmt = nodes.If(
496            nodes.Not(nodes.Name(name, "load")),
497            [nodes.Macro(name, m.args, m.defaults, m.body)],
498            [],
499            [],
500        )
501        return [m, if_stmt]