holoviz/panel · #8800
perf: Speed up streaming into HTML and Markdown panes
panel/models/html.ts300 + / 22 −
@@ -1,12 +1,12 @@ import {ModelEvent, server_event} from "@bokehjs/core/bokeh_events"+import {Signal} from "@bokehjs/core/signaling" import type {StyleSheetLike} from "@bokehjs/core/dom" import type * as p from "@bokehjs/core/properties" import type {Attrs, Dict} from "@bokehjs/core/types" import {entries} from "@bokehjs/core/util/object" import {Markup} from "@bokehjs/models/widgets/markup" import {PanelMarkupView} from "./layout" import {serializeEvent} from "./event-to-object"-import {throttle} from "./util" import html_css from "styles/models/html.css" @@ -28,25 +28,175 @@ function searchAllDOMs(node: Element | ShadowRoot, selector: string): (Element | return found } +/**+ * Replaces the characters between `start` and `end` of the unescaped+ * text with the escaped `patch`, producing `version` of the text.+ */ @server_event("html_stream") export class HTMLStreamEvent extends ModelEvent {- constructor(readonly model: HTML, readonly patch: string, readonly start: number) {+ constructor(readonly model: HTML, readonly patch: string, readonly start: number,+ readonly end: number | null = null, readonly version: number | null = null) { super() this.patch = patch this.start = start+ this.end = end+ this.version = version this.origin = model } protected override get event_values(): Attrs {- return {model: this.origin, patch: this.patch, start: this.start}+ return {model: this.origin, patch: this.patch, start: this.start, end: this.end, version: this.version} } static override from_values(values: object) {- const {model, patch, start} = values as {model: HTML, patch: string, start: number}- return new HTMLStreamEvent(model, patch, start)+ type Values = {model: HTML, patch: string, start: number, end: number | null, version: number | null}+ const {model, patch, start, end, version} = values as Values+ return new HTMLStreamEvent(model, patch, start, end, version) } } +const ESCAPES: {[key: string]: string} = {"&": "&", "<": "<", ">": ">", "\"": """, "'": "'"}+const UNESCAPES: {[key: string]: string} = Object.fromEntries(Object.entries(ESCAPES).map(([k, v]) => [v, k]))++// Exact inverses of Python's html.escape, unlike html_decode, which+// parses arbitrary entities and normalizes newlines.+export function escape_html(text: string): string {+ return text.replace(/[&<>"']/g, (c) => ESCAPES[c])+}++export function unescape_html(text: string): string {+ return text.replace(/&(?:amp|lt|gt|quot|#x27);/g, (e) => UNESCAPES[e])+}++const VOID_TAGS = new Set([+ "area", "base", "br", "col", "embed", "hr", "img", "input", "link",+ "meta", "param", "source", "track", "wbr",+])+const RAW_TEXT_TAGS = new Set([+ "script", "style", "textarea", "title", "xmp", "iframe", "noembed",+ "noframes", "noscript", "plaintext",+])+const TAG_NAME = /[a-zA-Z][^\s/>]*/y++function find_tag_end(html: string, pos: number): number {+ let quote: string | null = null+ let prev = ""+ for (let i = pos; i < html.length; i++) {+ const c = html[i]+ if (quote != null) {+ if (c === quote) {+ quote = null+ }+ } else if ((c === "\"" || c === "'") && prev === "=") {+ quote = c+ } else if (c === ">") {+ return i + 1+ }+ if (c.trim() !== "") {+ prev = c+ }+ }+ return -1+}++/**+ * Splits HTML into top-level blocks, i.e. ranges ending where all+ * opened elements are closed again, starting the scan at `from`. HTML+ * that never closes an element (or ends mid-tag) yields one block for+ * the remainder, so malformed input degrades to a full re-render.+ */+export function split_blocks(html: string, from: number = 0): [number, number][] {+ const blocks: [number, number][] = []+ const stack: string[] = []+ let foreign = 0+ let start = from+ let i = from+ const boundary = () => {+ if (stack.length === 0) {+ blocks.push([start, i])+ start = i+ }+ }+ while (true) {+ const lt = html.indexOf("<", i)+ if (lt === -1) {+ break+ }+ i = lt+ const next = html[i + 1]+ if (next === "!" || next === "?") {+ const comment = html.startsWith("<!--", i)+ const close = comment ? html.indexOf("-->", i + 4) : html.indexOf(">", i + 2)+ if (close === -1) {+ break+ }+ i = close + (comment ? 3 : 1)+ boundary()+ continue+ }+ const closing = next === "/"+ TAG_NAME.lastIndex = i + (closing ? 2 : 1)+ const match = TAG_NAME.exec(html)+ if (match == null) {+ i += 1+ continue+ }+ const name = match[0].toLowerCase()+ const end = find_tag_end(html, TAG_NAME.lastIndex)+ if (end === -1) {+ break+ }+ i = end+ if (closing) {+ const index = stack.lastIndexOf(name)+ if (index !== -1) {+ for (const tag of stack.splice(index)) {+ if (tag === "svg" || tag === "math") {+ foreign--+ }+ }+ boundary()+ }+ } else if (VOID_TAGS.has(name) || (foreign > 0 && html[end - 2] === "/")) {+ boundary()+ } else if (foreign === 0 && RAW_TEXT_TAGS.has(name)) {+ const close_tag = new RegExp(`</${name}[\\s/>]`, "ig")+ close_tag.lastIndex = i+ const close = close_tag.exec(html)+ const close_end = close == null ? -1 : html.indexOf(">", close.index)+ if (close_end === -1) {+ break+ }+ i = close_end + 1+ boundary()+ } else {+ stack.push(name)+ if (name === "svg" || name === "math") {+ foreign+++ }+ }+ }+ if (start < html.length) {+ blocks.push([start, html.length])+ }+ return blocks+}++function select_all(nodes: Iterable<Node>, selector: string): Element[] {+ const found: Element[] = []+ for (const node of nodes) {+ if (node instanceof Element) {+ if (node.matches(selector)) {+ found.push(node)+ }+ found.push(...node.querySelectorAll(selector))+ }+ }+ return found+}++type Block = {end: number, nodes: ChildNode[]}+ export class DOMEvent extends ModelEvent { constructor(readonly node: string, readonly data: unknown) { super()@@ -67,7 +217,11 @@ export function html_decode(input: string): string | null { } export function run_scripts(node: Element): void {- for (const old_script of node.querySelectorAll("script")) {+ replace_scripts(node.querySelectorAll("script"))+}++function replace_scripts(scripts: Iterable<Element>): void {+ for (const old_script of scripts) { const new_script = document.createElement("script") for (const attr of old_script.attributes) { new_script.setAttribute(attr.name, attr.value)@@ -82,7 +236,13 @@ export function run_scripts(node: Element): void { export class HTMLView extends PanelMarkupView { declare model: HTML- _buffer: string | null = null++ // DOM nodes rendered from each top-level block of the streamed text,+ // or null if the container was rendered as a whole.+ protected _blocks: Block[] | null = null+ // Lowest offset of the text modified since the last DOM update+ protected _dirty: number | null = null+ protected _frame: number | null = null protected readonly _event_listeners: Map<string, Map<string, (event: Event) => void>> = new Map() @@ -91,7 +251,8 @@ export class HTMLView extends PanelMarkupView { const {text, visible, events} = this.model.properties this.on_change(text, () => {- this._buffer = null+ this.model.reset_stream()+ this._dirty = null const html = this.process_tex() this.set_html(html) })@@ -105,34 +266,102 @@ export class HTMLView extends PanelMarkupView { this._setup_event_listeners() }) - const set_text = throttle(() => {- const text = this._buffer- this._buffer = null- this.model.setv({text}, {silent: true})- const html = this.process_tex()- this.set_html(html)- }, 10)- this.model.on_event(HTMLStreamEvent, (event: HTMLStreamEvent) => {- const beginning = this._buffer == null ? this.model.text : this._buffer- this._buffer = beginning.slice(0, event.start)+event.patch- set_text()+ this.model.on_event(HTMLStreamEvent, (event: HTMLStreamEvent) => this.model.apply_stream(event))+ this.connect(this.model.streamed, (start: number) => {+ this._dirty = Math.min(this._dirty ?? start, start)+ if (this._frame == null) {+ this._frame = requestAnimationFrame(() => this._flush_stream())+ } }) } + override remove(): void {+ if (this._frame != null) {+ cancelAnimationFrame(this._frame)+ this._frame = null+ }+ super.remove()+ }+ override stylesheets(): StyleSheetLike[] { return [...super.stylesheets(), html_css] } + protected _flush_stream(): void {+ this._frame = null+ const dirty = this._dirty+ this._dirty = null+ if (dirty == null || this.container == null) {+ return+ }+ const html = this.model.stream_text+ if (!this.model.disable_math && this.contains_tex(html)) {+ this.set_html(this.process_tex())+ } else {+ this._update_blocks(html, dirty)+ }+ }++ /**+ * Re-renders only the top-level blocks at or after the first+ * modified offset, keeping the DOM (and any text selection) of the+ * preceding blocks intact.+ */+ protected _update_blocks(html: string, dirty: number): void {+ let blocks = this._blocks+ let keep = 0+ if (blocks == null) {+ this.container.replaceChildren()+ blocks = []+ } else {+ // The last block may still be growing even if it ends before dirty+ while (keep < blocks.length - 1 && blocks[keep].end <= dirty) {+ keep+++ }+ for (const block of blocks.splice(keep)) {+ for (const node of block.nodes) {+ node.remove()+ }+ }+ }+ const from = keep > 0 ? blocks[keep - 1].end : 0+ const fragment = document.createDocumentFragment()+ const parser = document.createElement("div")+ const added: ChildNode[] = []+ for (const [start, end] of split_blocks(html, from)) {+ parser.innerHTML = html.slice(start, end)+ const nodes = [...parser.childNodes]+ fragment.append(...nodes)+ added.push(...nodes)+ blocks.push({end, nodes})+ }+ this.container.append(fragment)+ this._blocks = blocks+ if (this.model.run_scripts) {+ replace_scripts(select_all(added, "script"))+ }+ if (Object.keys(this.model.events).length !== 0) {+ this._remove_event_listeners()+ this._setup_event_listeners()+ }+ this._decorate(added)+ }+ set_html(html: string | null): void { if (html === null || this.container == null) { return }+ this._blocks = null this.container.innerHTML = html if (this.model.run_scripts) { run_scripts(this.container) } this._setup_event_listeners()- for (const codeblock of this.container.querySelectorAll(".codehilite")) {+ this._decorate([this.container])+ }++ protected _decorate(nodes: Iterable<Node>): void {+ for (const codeblock of select_all(nodes, ".codehilite")) { const copy_button = document.createElement("button") const pre = (codeblock.children[0] as HTMLPreElement) copy_button.className = "copybtn"@@ -148,7 +377,7 @@ export class HTMLView extends PanelMarkupView { }) codeblock.insertBefore(copy_button, pre) }- for (const anchor of this.container.querySelectorAll("a")) {+ for (const anchor of select_all(nodes, "a")) { const link = anchor.getAttribute("href") if (link && link.startsWith("#")) { anchor.addEventListener("click", () => {@@ -262,6 +491,7 @@ export namespace HTML { export type Props = Markup.Props & { events: p.Property<Dict<string[]>> run_scripts: p.Property<boolean>+ stream_version: p.Property<number> } } @@ -270,17 +500,65 @@ export interface HTML extends HTML.Attrs {} export class HTML extends Markup { declare properties: HTML.Props + // Unescaped text maintained from stream events and the events that+ // arrived ahead of the next version. Views share the model, so events+ // are applied here, once and in order, rather than by each view.+ protected _stream_text: string | null = null+ protected _stream_queue: Map<number, HTMLStreamEvent> = new Map()++ // Emits the lowest offset of the text modified by a stream event+ readonly streamed = new Signal<number, this>(this, "streamed")+ constructor(attrs?: Partial<HTML.Attrs>) { super(attrs) } + get stream_text(): string {+ return this._stream_text ??= unescape_html(this.text)+ }++ apply_stream(event: HTMLStreamEvent): void {+ if (event.version == null) {+ this._apply_stream(event)+ return+ }+ if (event.version <= this.stream_version) {+ return+ }+ // Messages written to the socket via different paths on the+ // server may arrive out of order.+ this._stream_queue.set(event.version, event)+ let next: HTMLStreamEvent | undefined+ while ((next = this._stream_queue.get(this.stream_version + 1)) != null) {+ this._stream_queue.delete(next.version!)+ this._apply_stream(next)+ }+ }++ protected _apply_stream(event: HTMLStreamEvent): void {+ const text = this.stream_text+ const end = event.end ?? text.length+ this._stream_text = text.slice(0, event.start) + unescape_html(event.patch) + text.slice(end)+ const attrs: Partial<HTML.Attrs> = {text: escape_html(this._stream_text)}+ if (event.version != null) {+ attrs.stream_version = event.version+ }+ this.setv(attrs, {silent: true})+ this.streamed.emit(event.start)+ }++ reset_stream(): void {+ this._stream_text = null+ }+ static override __module__ = "panel.models.markup" static { this.prototype.default_view = HTMLView- this.define<HTML.Props>(({Bool, Str, List, Dict}) => ({+ this.define<HTML.Props>(({Bool, Int, Str, List, Dict}) => ({ events: [ Dict(List(Str)), {} ], run_scripts: [ Bool, true ],+ stream_version: [ Int, 0 ], })) } }panel/models/markup.py16 + / 2 −
@@ -12,16 +12,26 @@ class HTMLStreamEvent(ModelEvent):+ """+ Replaces the characters between start and end of the unescaped+ text with the (escaped) patch, producing the given stream_version+ of the text.+ """ event_name = 'html_stream' - def __init__(self, model, patch=None, start=None):+ def __init__(self, model, patch=None, start=None, end=None, version=None): self.patch = patch self.start = start+ self.end = end+ self.version = version super().__init__(model=model) def event_values(self) -> dict[str, t.Any]:- return dict(super().event_values(), patch=self.patch, start=self.start)+ return dict(+ super().event_values(), patch=self.patch, start=self.start,+ end=self.end, version=self.version+ ) class HTML(Markup):@@ -33,6 +43,10 @@ class HTML(Markup): run_scripts = Bool(True, help="Whether to run scripts defined within the HTML") + stream_version = Int(0, help="""+ Number of HTMLStreamEvents applied to the text, allowing the+ frontend to apply events in order and exactly once.""")+ class JSON(Markup): """panel/pane/markup.py211 + / 25 −
@@ -4,28 +4,42 @@ """ from __future__ import annotations +import asyncio import functools import json+import re import textwrap+import time import typing as t -from html import escape+from functools import partial+from html import escape, unescape import param # type: ignore from ..config import config+from ..io.document import unlocked+from ..io.notebook import push from ..io.resources import CDN_DIST+from ..io.state import state from ..models.markup import HTML as _BkHTML, JSON as _BkJSON, HTMLStreamEvent-from ..util import HTML_SANITIZER, prefix_length+from ..util import HTML_SANITIZER, splice_diff, utf16_offset from .base import ModelPane if t.TYPE_CHECKING: from collections.abc import Mapping from bokeh.document import Document from bokeh.model import Model+ from markdown_it import MarkdownIt from pyviz_comms import Comm # type: ignore +# Syntax whose rendering depends on other blocks (footnotes, reference+# links, definition lists) or which breaks line offsets (\r).+_NON_LOCAL_SYNTAX = re.compile(r'\[\^|\]:|^ {0,3}[:~][ \t]|\r', re.MULTILINE)++_HEADING_ID = re.compile(r'<h[1-6][^>]*?\sid="([^"]*)"')+ class HTMLBasePane(ModelPane): """@@ -44,18 +58,135 @@ class HTMLBasePane(ModelPane): _updates: t.ClassVar[bool] = True + # Minimum interval (in seconds) between streamed updates sent to a+ # view; updates arriving in between are coalesced into one patch.+ _stream_interval: t.ClassVar[float] = 0.03+ __abstract = True + def __init__(self, object=None, **params):+ self._raw_html: str | None = None+ # Per view: (escaped text, unescaped text) last sent to the frontend+ self._stream_state: dict[str, tuple[str, str]] = {}+ # Per view: token identifying the scheduled flush, or its due time+ self._stream_pending: dict[str, float] = {}+ self._stream_last: dict[str, float] = {}+ super().__init__(object=object, **params)++ def _escape_html(self, html: str) -> dict[str, t.Any]:+ # Keeping the unescaped HTML saves unescaping it when streaming+ self._raw_html = html+ return dict(object=escape(html))++ def _cleanup(self, root: Model | None = None) -> None:+ if root is not None:+ ref = root.ref['id']+ self._stream_state.pop(ref, None)+ self._stream_pending.pop(ref, None)+ self._stream_last.pop(ref, None)+ super()._cleanup(root)++ def _update_pane(self, *events) -> None:+ if (not self.enable_streaming or not self._stream_interval or+ any(event.name != 'object' for event in events)):+ super()._update_pane(*events)+ return+ for ref in list(self._models):+ if ref not in state._views or ref in state._fake_roots:+ continue+ doc, comm = state._views[ref][2:]+ if comm or not doc.session_context or state._unblocked(doc):+ self._stream_update(ref)+ elif ref not in self._stream_pending:+ # Off the event loop thread the throttle has to be+ # applied on the loop, which picks up the latest object.+ self._stream_pending[ref] = token = time.monotonic()+ doc.add_next_tick_callback(state._handle_exception_wrapper(+ partial(self._stream_scheduled, ref, token, True), doc+ ))++ def _stream_update(self, ref: str) -> None:+ if ref not in self._models or ref not in state._views:+ return+ doc, comm = state._views[ref][2:]+ now = time.monotonic()+ due = self._stream_pending.get(ref)+ if due is not None and now < due:+ return+ wait = self._stream_last.get(ref, -float('inf')) + self._stream_interval - now+ if due is None and wait > 0 and self._stream_schedule(ref, doc, comm, now + wait):+ return+ self._stream_flush(ref)++ def _stream_schedule(self, ref: str, doc: Document, comm: Comm | None, due: float) -> bool:+ """+ Schedules a trailing flush, returning False if no event loop+ is available to run it.+ """+ callback = state._handle_exception_wrapper(+ partial(self._stream_scheduled, ref, due), doc+ )+ delay = max(due - time.monotonic(), 0)+ if doc.session_context:+ doc.add_timeout_callback(callback, int(delay * 1000))+ elif comm:+ try:+ loop = asyncio.get_running_loop()+ except RuntimeError:+ return False+ loop.call_later(delay, callback)+ else:+ return False+ self._stream_pending[ref] = due+ return True++ def _stream_scheduled(self, ref: str, token: float, throttle: bool = False) -> None:+ # A blocked event loop may have forced an earlier flush, which+ # superseded this callback.+ if self._stream_pending.get(ref) != token:+ return+ del self._stream_pending[ref]+ if throttle:+ self._stream_update(ref)+ else:+ self._stream_flush(ref)++ def _stream_flush(self, ref: str) -> None:+ self._stream_pending.pop(ref, None)+ if ref not in self._models or ref not in state._views:+ return+ _, root, doc, comm = state._views[ref]+ self._stream_last[ref] = time.monotonic()+ with unlocked():+ self._update_object(ref, doc, root, self._models[ref][1], comm)+ if comm and 'embedded' not in root.tags:+ push(doc, comm)+ def _update(self, ref: str, model: Model) -> None:+ self._raw_html = None props = self._get_properties(model.document)- if self.enable_streaming and 'text' in props:- text = props['text']- start = prefix_length(text, model.text)- model.run_scripts = False- patch = text[start:]- self._send_event(HTMLStreamEvent, patch=patch, start=start)- model._property_values['text'] = model.text[:start]+patch- del props['text']+ text = props.get('text')+ if (not self.enable_streaming or text is None or ref not in state._views+ or not isinstance(model, _BkHTML)):+ model.update(**props)+ return+ del props['text']+ html = unescape(text) if self._raw_html is None else self._raw_html+ old_text, old_html = self._stream_state.get(ref, (None, None))+ if old_text is not model.text or old_html is None:+ old_html = unescape(model.text)+ if html != old_html:+ start, end, patch = splice_diff(old_html, html)+ start, end = utf16_offset(old_html, start), utf16_offset(old_html, end)+ version = model._property_values.get('stream_version', 0) + 1+ props['run_scripts'] = False+ # Bypass the property setters so no full text update is sent+ model._property_values['text'] = text+ model._property_values['stream_version'] = version+ state._views[ref][2].callbacks.send_event(HTMLStreamEvent(+ model=model, patch=escape(patch), start=start, end=end, version=version+ ))+ self._stream_state[ref] = (model.text, html) model.update(**props) @@ -114,7 +245,7 @@ def _transform_object(self, obj: t.Any) -> dict[str, t.Any]: text = text._repr_html_() if self.sanitize_html: text = self.sanitize_hook(text)- return dict(object=escape(text))+ return self._escape_html(text) class DataFrame(HTML):@@ -395,7 +526,7 @@ def _transform_object(self, obj: t.Any) -> dict[str, t.Any]: html = obj.to_html(**kwargs) else: html = ''- return dict(object=escape(html))+ return self._escape_html(html) def _init_params(self) -> dict[str, t.Any]: params = HTMLBasePane._init_params(self)@@ -443,7 +574,7 @@ def _transform_object(self, obj: t.Any) -> dict[str, t.Any]: text = '<pre> </pre>' else: text = '<pre>'+str(obj)+'</pre>'- return dict(object=escape(text))+ return self._escape_html(text) class Markdown(HTMLBasePane):@@ -575,8 +706,15 @@ def hilite(token, langname, attrs): parser.options['highlight'] = hilite return parser + def __init__(self, object=None, **params):+ # (parser, source, html, heading slugs) of the leading blocks+ self._block_cache: tuple[MarkdownIt, str, str, frozenset[str]] | None = None+ super().__init__(object=object, **params)+ def _transform_object(self, obj: t.Any) -> dict[str, t.Any]:- import markdown+ return self._escape_html(self._render_markdown(obj))++ def _render_markdown(self, obj: t.Any) -> str: if obj is None: obj = '' elif not isinstance(obj, str):@@ -585,25 +723,73 @@ def _transform_object(self, obj: t.Any) -> dict[str, t.Any]: obj = textwrap.dedent(obj) if self.renderer == 'markdown':+ import markdown+ self._block_cache = None extensions = self.extensions + ['nl2br'] if self.hard_line_break else self.extensions- html = markdown.markdown(+ return markdown.markdown( obj, extensions=extensions, output_format='xhtml', **self.renderer_options )- else:- parser = self._get_parser(- self.renderer, tuple(self.plugins), self.hard_line_break, self.disable_anchors, **self.renderer_options- )+ parser = self._get_parser(+ self.renderer, tuple(self.plugins), self.hard_line_break, self.disable_anchors, **self.renderer_options+ )+ if self.renderer == 'markdown-it' and not self.plugins and not _NON_LOCAL_SYNTAX.search(obj): try:- html = parser.render(obj)+ return self._render_incremental(parser, obj) except IndexError:- # Likely markdown-it mdurl parser error- with parser.reset_rules():- parser.disable('link')- html = parser.render(obj)- return dict(object=escape(html))+ pass+ self._block_cache = None+ try:+ return parser.render(obj)+ except IndexError:+ # Likely markdown-it mdurl parser error+ with parser.reset_rules():+ parser.disable('link')+ return parser.render(obj)++ def _render_incremental(self, parser: MarkdownIt, src: str) -> str:+ """+ Renders markdown reusing the HTML of the leading top-level+ blocks from the previous render if the source was appended to.+ Every block but the last is final once a later block starts, so+ streaming only has to re-parse and re-highlight the last block.+ """+ cache = self._block_cache+ if cache and cache[0] is parser and src.startswith(cache[1]):+ _, prefix_src, prefix_html, slugs = cache+ else:+ prefix_src, prefix_html, slugs = '', '', frozenset()+ tail_src = src[len(prefix_src):]+ env: dict[str, t.Any] = {}+ tokens = parser.parse(tail_src, env)+ # Index and first source line of the last top-level block+ split, line = next((+ (i, token.map[0]) for i in range(len(tokens)-1, 0, -1)+ if (token := tokens[i]).level == 0 and token.map is not None+ ), (0, 0))+ render = parser.renderer.render+ stable_html = render(tokens[:split], parser.options, env) if split else ''+ tail_html = render(tokens[split:], parser.options, env)+ if slugs and not self.disable_anchors:+ # Heading anchors are deduplicated per render+ new_slugs = _HEADING_ID.findall(stable_html + tail_html)+ if not slugs.isdisjoint(new_slugs):+ self._block_cache = None+ return self._render_incremental(parser, src)+ if split:+ offset = 0+ for _ in range(line):+ offset = tail_src.index('\n', offset) + 1+ if not self.disable_anchors:+ slugs = slugs.union(_HEADING_ID.findall(stable_html))+ self._block_cache = (+ parser, prefix_src + tail_src[:offset], prefix_html + stable_html, slugs+ )+ else:+ self._block_cache = (parser, prefix_src, prefix_html, slugs)+ return prefix_html + stable_html + tail_html def _process_param_change(self, params): if 'css_classes' in params:panel/tests/pane/test_markup.py135 + / 0 −
@@ -499,3 +499,138 @@ def test_json_theme(): with patch('panel.config._config.theme', new_callable=lambda: "dark"): assert JSON({"x": 1}, theme="light").theme == "light"+++STREAM_DOCS = [+ "# Title\n\nSome *text* with `code`.\n\n- a\n- b\n\n - nested\n\n```python\nx = 1\n```\n\n| a | b |\n|---|---|\n| 1 | 2 |\n",+ "para\n- list interrupts\n\n> quote\nlazy\n\n indented\n\n code\n\nsetext\n===\n\n<div>\n\n*html*\n\n</div>\n\n```\nunclosed",+ "# A\n\nx\n\n## A\n\ny\n\n# A\n",+]++@pytest.mark.parametrize('src', STREAM_DOCS)+@pytest.mark.parametrize('disable_anchors', [False, True])+def test_markdown_incremental_render_matches_full_render(src, disable_anchors):+ pane = Markdown(disable_anchors=disable_anchors, dedent=False)+ parser = pane._get_parser(pane.renderer, (), False, disable_anchors)+ for i in range(1, len(src)+1, 3):+ assert pane._render_markdown(src[:i]) == parser.render(src[:i])+ assert pane._render_markdown(src) == parser.render(src)++def test_markdown_incremental_render_caches_leading_blocks():+ pane = Markdown(dedent=False)+ pane._render_markdown("a\n\nb\n\nc")+ assert pane._block_cache[1:3] == ("a\n\nb\n\n", "<p>a</p>\n<p>b</p>\n")+ assert pane._render_markdown("a\n\nb\n\nc *d*") == "<p>a</p>\n<p>b</p>\n<p>c <em>d</em></p>\n"++def test_markdown_incremental_render_resets_on_replace():+ pane = Markdown(dedent=False)+ pane._render_markdown("a\n\nb\n\nc")+ assert pane._render_markdown("x\n\ny") == "<p>x</p>\n<p>y</p>\n"++@pytest.mark.parametrize('src', [+ "See [^1].\n\nPara\n\n[^1]: Note\n",+ "See [link].\n\nPara\n\n[link]: https://example.com\n",+ "Term\n\n: Definition\n",+])+def test_markdown_non_local_syntax_renders_fully(src):+ pane = Markdown(dedent=False)+ parser = pane._get_parser(pane.renderer, (), False, False)+ for i in range(1, len(src)+1):+ assert pane._render_markdown(src[:i]) == parser.render(src[:i])+ assert pane._block_cache is None++def _stream_events(doc):+ return patch.object(doc.callbacks, 'send_event')++def test_html_stream_sends_splice(document, comm):+ pane = Markdown('Hello', enable_streaming=True)+ model = pane.get_root(document, comm=comm)+ with _stream_events(document) as send_event:+ pane.object = 'Hello *world*'+ send_event.assert_called_once()+ event = send_event.call_args.args[0]+ assert event.model is model+ assert (event.start, event.end) == (8, 8)+ assert event.patch == html.escape(' <em>world</em>')+ assert html.unescape(model.text) == '<p>Hello <em>world</em></p>\n'+ assert model.run_scripts is False++def test_html_stream_offsets_count_utf16_units(document, comm):+ pane = HTML('<p>😀 a</p>', enable_streaming=True)+ pane.get_root(document, comm=comm)+ with _stream_events(document) as send_event:+ pane.object = '<p>😀 ab</p>'+ event = send_event.call_args.args[0]+ # The emoji is a single code point but two UTF-16 code units+ assert (event.start, event.end, event.patch) == (7, 7, 'b')++def test_html_stream_skips_unchanged(document, comm):+ pane = Markdown('Hello', enable_streaming=True)+ pane.get_root(document, comm=comm)+ with _stream_events(document) as send_event:+ pane.object = 'Hello'+ pane.dedent = False+ send_event.assert_not_called()++def test_html_stream_sends_event_only_to_own_view(comm):+ from bokeh.document import Document+ pane = HTML('<b>a</b>', enable_streaming=True)+ docs = [Document(), Document()]+ models = [pane.get_root(doc, comm=comm) for doc in docs]+ with _stream_events(docs[0]) as send0, _stream_events(docs[1]) as send1:+ pane.object = '<b>ab</b>'+ for send, model in zip((send0, send1), models):+ send.assert_called_once()+ assert send.call_args.args[0].model is model+ assert html.unescape(model.text) == '<b>ab</b>'++def test_html_stream_disabled_sends_text(document, comm):+ pane = Markdown('Hello')+ model = pane.get_root(document, comm=comm)+ with _stream_events(document) as send_event:+ pane.object = 'Hello *world*'+ send_event.assert_not_called()+ assert html.unescape(model.text) == '<p>Hello <em>world</em></p>\n'++async def test_html_stream_coalesces_updates(document, comm):+ pane = Markdown('', enable_streaming=True)+ model = pane.get_root(document, comm=comm)+ with _stream_events(document) as send_event:+ for i in range(10):+ pane.object += str(i)+ assert send_event.call_count == 1+ await asyncio.sleep(pane._stream_interval * 3)+ assert send_event.call_count == 2+ assert html.unescape(model.text) == '<p>0123456789</p>\n'++async def test_html_stream_flushes_when_loop_blocked(document, comm):+ import time+ pane = Markdown('', enable_streaming=True)+ model = pane.get_root(document, comm=comm)+ with _stream_events(document) as send_event:+ for i in range(3):+ pane.object += str(i)+ time.sleep(pane._stream_interval * 1.5)+ assert send_event.call_count == 3+ await asyncio.sleep(pane._stream_interval * 3)+ assert send_event.call_count == 3+ assert html.unescape(model.text) == '<p>012</p>\n'++@pytest.mark.parametrize('enable_streaming', [False, True])+def test_html_base_pane_subclass_without_transform(document, comm, enable_streaming):+ from panel.pane.markup import HTMLBasePane++ class Text(str):+ pass++ # Pane types are registered globally, so only apply to a private type+ class Custom(HTMLBasePane):+ @classmethod+ def applies(cls, obj):+ return isinstance(obj, Text)++ pane = Custom(Text('<b>a</b>'), enable_streaming=enable_streaming)+ model = pane.get_root(document, comm=comm)+ assert model.text == '<b>a</b>'+ pane.object = Text('<b>ab</b>')+ assert model.text == '<b>ab</b>'panel/tests/test_util.py50 + / 1 −
@@ -1,6 +1,7 @@ from collections import OrderedDict import param+import pytest from bokeh.models import Div @@ -10,7 +11,7 @@ from panel.tests.util import mpl_available from panel.util import ( abbreviated_repr, extract_dependencies, get_method_owner, parse_query,- styler_update,+ splice_diff, styler_update, suffix_length, utf16_offset, ) @@ -143,3 +144,51 @@ def test_styler_update(dataframe): (2, 0): [('background-color', '#67000d'), ('color', '#f1f1f1')], (2, 1): [('background-color', '#67000d'), ('color', '#f1f1f1')] }+++@pytest.mark.parametrize('a, b, expected', [+ ('', '', 0),+ ('abc', '', 0),+ ('abc', 'xbc', 2),+ ('abc', 'abc', 3),+ ('x' * 200 + 'abc', 'y' * 300 + 'abc', 3),+ ('x' * 200, 'y' + 'x' * 150, 150),+])+def test_suffix_length(a, b, expected):+ assert suffix_length(a, b) == expected+++def test_suffix_length_limit():+ assert suffix_length('aaaa', 'aaaa', 2) == 2+++@pytest.mark.parametrize('old, new', [+ ('', 'abc'),+ ('abc', ''),+ ('<p>Hello</p>', '<p>Hello world</p>'),+ ('<p>a</p>', '<p>a</p><p>b</p>'),+ ('aaaa', 'aaaaaa'),+ ('abcabc', 'abc'),+ ('same', 'same'),+ ('<ul><li>a</li></ul>', '<ul><li>a</li><li>b</li></ul>'),+])+def test_splice_diff(old, new):+ start, end, patch = splice_diff(old, new)+ assert old[:start] + patch + old[end:] == new+ assert start <= end <= len(old)+ assert len(patch) == len(new) - (len(old) - (end - start))+++def test_splice_diff_minimal_patch_before_closing_tags():+ assert splice_diff('<p>Hello</p>\n', '<p>Hello world</p>\n') == (8, 8, ' world')+++@pytest.mark.parametrize('text, index, expected', [+ ('abc', 2, 2),+ ('é😀a', 1, 1),+ ('é😀a', 2, 3),+ ('😀😀a', 3, 5),+])+def test_utf16_offset(text, index, expected):+ assert utf16_offset(text, index) == expected+ assert len(text[:index].encode('utf-16-le')) // 2 == expectedpanel/tests/ui/pane/test_markup.py77 + / 1 −
@@ -8,7 +8,8 @@ from panel.layout import Column, Row from panel.models import HTML-from panel.pane import Markdown+from panel.models.markup import HTMLStreamEvent+from panel.pane import HTML as HTMLPane, Markdown from panel.tests.util import serve_component, wait_until from panel.widgets import Button @@ -108,6 +109,81 @@ def test_markdown_pane_stream(page): expect(page.locator('.markdown')).to_have_text(md.object) +def test_markdown_pane_stream_keeps_completed_blocks(page):+ md = Markdown('# Title', enable_streaming=True)++ serve_component(page, md)++ # The first streamed update re-renders the initial (non-streamed) DOM+ md.object += '\n\nFirst paragraph'+ expect(page.locator('.markdown p')).to_have_text('First paragraph')+ page.locator('.markdown h1').evaluate("el => { el.dataset.marker = 'kept' }")++ for i in range(20):+ md.object += f' word{i}'+ md.object += '\n\nSecond paragraph'++ expect(page.locator('.markdown p').nth(1)).to_have_text('Second paragraph')+ expect(page.locator('.markdown h1')).to_have_attribute('data-marker', 'kept')+ expect(page.locator('.markdown p').first).to_have_text(+ 'First paragraph ' + ' '.join(f'word{i}' for i in range(20))+ )+++def test_markdown_pane_stream_matches_full_render(page):+ src = (+ "## Heading\n\nSome *text* & <b>html</b> with `code`.\n\n"+ "- item\n - nested\n\n```python\nx = {'a': [1, 2]}\n```\n\n"+ "| a | b |\n|---|---|\n| 1 | 2 |\n"+ )+ streamed = Markdown('', enable_streaming=True, css_classes=['streamed'])+ reference = Markdown(src, css_classes=['reference'])++ serve_component(page, Column(streamed, reference))++ for i in range(0, len(src), 3):+ streamed.object = src[:i+3]++ inner_html = "el => [...el.shadowRoot.children].filter(c => c.tagName === 'DIV').pop().innerHTML"+ expect(page.locator('.streamed table')).to_be_visible()+ wait_until(lambda: (+ page.locator('.streamed').evaluate(inner_html) ==+ page.locator('.reference').evaluate(inner_html)+ ), page)+ expect(page.locator('.streamed .copybtn')).to_have_count(1)+++def test_markdown_pane_stream_emoji(page):+ md = Markdown('', enable_streaming=True, disable_anchors=True)++ serve_component(page, md)++ expected = ''+ for token in ['Hi 😀', ' there', ' 🎉🎉', ' **bold** 👍', ' end']:+ md.object += token+ expected += token+ page.wait_for_timeout(50)++ expect(page.locator('.markdown p')).to_have_text('Hi 😀 there 🎉🎉 bold 👍 end')+ expect(page.locator('.markdown strong')).to_have_text('bold')+++def test_html_pane_stream_events_applied_in_order(page):+ pane = HTMLPane('<p>a</p>', enable_streaming=True)++ serve_component(page, pane)++ expect(page.locator('p')).to_have_text('a')++ # Event 2 arrives before event 1 and is only applied after it+ pane._send_event(HTMLStreamEvent, patch=escape('c'), start=5, end=5, version=2)+ pane._send_event(HTMLStreamEvent, patch=escape('b'), start=4, end=4, version=1)+ # Replayed events are ignored+ pane._send_event(HTMLStreamEvent, patch=escape('b'), start=4, end=4, version=1)++ expect(page.locator('p')).to_have_text('abc')++ def test_html_model_no_stylesheet(page): # regression test for https://github.com/holoviz/holoviews/issues/5963 text = "<h1>Header</h1>"panel/util/__init__.py56 + / 0 −
@@ -561,6 +561,62 @@ def prefix_length(a: str, b: str) -> int: return left +def suffix_length(a: str, b: str, limit: int | None = None) -> int:+ """+ Searches for the length of the common suffix of strings a and b,+ optionally capped at limit characters, using binary search.+ """+ la, lb = len(a), len(b)+ right = min(la, lb)+ if limit is not None:+ right = min(right, limit)+ # Common suffixes are usually short (closing tags), so gallop up+ # from a small window to avoid slicing large parts of the strings.+ left, probe = 0, 64+ while probe < right and a[la-probe:] == b[lb-probe:]:+ left, probe = probe, probe * 2+ right = min(right, probe)+ while left < right:+ mid = (left + right + 1) // 2+ if a[la-mid:] == b[lb-mid:]:+ left = mid+ else:+ right = mid - 1+ return left+++_ASTRAL = re.compile('[\U00010000-\U0010FFFF]')++def utf16_offset(text: str, index: int) -> int:+ """+ Converts a code point index into text to the UTF-16 code unit+ index JavaScript uses, where astral characters (e.g. emoji) take+ up two units.+ """+ if text.isascii():+ return index+ return index + len(_ASTRAL.findall(text, 0, index))+++def splice_diff(old: str, new: str) -> tuple[int, int, str]:+ """+ Computes a single splice transforming old into new, such that+ `new == old[:start] + patch + old[end:]`.++ Returns+ -------+ start: int+ Length of the common prefix.+ end: int+ Index in old where the common suffix begins.+ patch: str+ The replacement for old[start:end].+ """+ start = prefix_length(new, old)+ suffix = suffix_length(new, old, min(len(old), len(new)) - start)+ return start, len(old) - suffix, new[start:len(new)-suffix]++ def camel_to_kebab(name): # Use regular expressions to insert a hyphen before each uppercase letter not at the start, # and between a lowercase and uppercase letter.