Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cleanup markdown span handling, take 2 #80550

Merged
merged 5 commits into from
Jan 2, 2021
Merged

Cleanup markdown span handling, take 2 #80550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
08f7b0a
Use ranges returned by pulldown
bugadani Dec 31, 2020
2b70da1
Remove unnecessary scope
bugadani Dec 31, 2020
cb4317d
Always provide a range
bugadani Dec 31, 2020
854b9d1
Collect links into a single vector
bugadani Dec 31, 2020
f073543
Only use locate for borrowed strings
bugadani Dec 31, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 84 additions & 71 deletions src/librustdoc/html/markdown.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use rustc_session::lint;
use rustc_span::edition::Edition;
use rustc_span::Span;
use std::borrow::Cow;
use std::cell::RefCell;
use std::collections::VecDeque;
use std::default::Default;
use std::fmt::Write;
Expand Down Expand Up @@ -414,11 +415,13 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for LinkReplacer<'a, I> {
}
}

type SpannedEvent<'a> = (Event<'a>, Range<usize>);

/// Make headings links with anchor IDs and build up TOC.
struct HeadingLinks<'a, 'b, 'ids, I> {
inner: I,
toc: Option<&'b mut TocBuilder>,
buf: VecDeque<Event<'a>>,
buf: VecDeque<SpannedEvent<'a>>,
id_map: &'ids mut IdMap,
}

Expand All @@ -428,48 +431,48 @@ impl<'a, 'b, 'ids, I> HeadingLinks<'a, 'b, 'ids, I> {
}
}

impl<'a, 'b, 'ids, I: Iterator<Item = Event<'a>>> Iterator for HeadingLinks<'a, 'b, 'ids, I> {
type Item = Event<'a>;
impl<'a, 'b, 'ids, I: Iterator<Item = SpannedEvent<'a>>> Iterator
for HeadingLinks<'a, 'b, 'ids, I>
{
type Item = SpannedEvent<'a>;

fn next(&mut self) -> Option<Self::Item> {
if let Some(e) = self.buf.pop_front() {
return Some(e);
}

let event = self.inner.next();
if let Some(Event::Start(Tag::Heading(level))) = event {
if let Some((Event::Start(Tag::Heading(level)), _)) = event {
let mut id = String::new();
for event in &mut self.inner {
match &event {
match &event.0 {
Event::End(Tag::Heading(..)) => break,
Event::Start(Tag::Link(_, _, _)) | Event::End(Tag::Link(..)) => {}
Event::Text(text) | Event::Code(text) => {
id.extend(text.chars().filter_map(slugify));
self.buf.push_back(event);
}
_ => {}
}
match event {
Event::Start(Tag::Link(_, _, _)) | Event::End(Tag::Link(..)) => {}
event => self.buf.push_back(event),
_ => self.buf.push_back(event),
}
}
let id = self.id_map.derive(id);

if let Some(ref mut builder) = self.toc {
let mut html_header = String::new();
html::push_html(&mut html_header, self.buf.iter().cloned());
html::push_html(&mut html_header, self.buf.iter().map(|(ev, _)| ev.clone()));
let sec = builder.push(level as u32, html_header, id.clone());
self.buf.push_front(Event::Html(format!("{} ", sec).into()));
self.buf.push_front((Event::Html(format!("{} ", sec).into()), 0..0));
}

self.buf.push_back(Event::Html(format!("</a></h{}>", level).into()));
self.buf.push_back((Event::Html(format!("</a></h{}>", level).into()), 0..0));

let start_tags = format!(
"<h{level} id=\"{id}\" class=\"section-header\">\
<a href=\"#{id}\">",
id = id,
level = level
);
return Some(Event::Html(start_tags.into()));
return Some((Event::Html(start_tags.into()), 0..0));
}
event
}
Expand Down Expand Up @@ -560,23 +563,23 @@ impl<'a, I> Footnotes<'a, I> {
}
}

impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
type Item = Event<'a>;
impl<'a, I: Iterator<Item = SpannedEvent<'a>>> Iterator for Footnotes<'a, I> {
type Item = SpannedEvent<'a>;

fn next(&mut self) -> Option<Self::Item> {
loop {
match self.inner.next() {
Some(Event::FootnoteReference(ref reference)) => {
Some((Event::FootnoteReference(ref reference), range)) => {
let entry = self.get_entry(&reference);
let reference = format!(
"<sup id=\"fnref{0}\"><a href=\"#fn{0}\">{0}</a></sup>",
(*entry).1
);
return Some(Event::Html(reference.into()));
return Some((Event::Html(reference.into()), range));
}
Some(Event::Start(Tag::FootnoteDefinition(def))) => {
Some((Event::Start(Tag::FootnoteDefinition(def)), _)) => {
let mut content = Vec::new();
for event in &mut self.inner {
for (event, _) in &mut self.inner {
if let Event::End(Tag::FootnoteDefinition(..)) = event {
break;
}
Expand Down Expand Up @@ -607,7 +610,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
ret.push_str("</li>");
}
ret.push_str("</ol></div>");
return Some(Event::Html(ret.into()));
return Some((Event::Html(ret.into()), 0..0));
} else {
return None;
}
Expand Down Expand Up @@ -917,13 +920,14 @@ impl Markdown<'_> {
};

let p = Parser::new_with_broken_link_callback(md, opts(), Some(&mut replacer));
let p = p.into_offset_iter();

let mut s = String::with_capacity(md.len() * 3 / 2);

let p = HeadingLinks::new(p, None, &mut ids);
let p = LinkReplacer::new(p, links);
let p = CodeBlocks::new(p, codes, edition, playground);
let p = Footnotes::new(p);
let p = LinkReplacer::new(p.map(|(ev, _)| ev), links);
let p = CodeBlocks::new(p, codes, edition, playground);
html::push_html(&mut s, p);

s
Expand All @@ -934,16 +938,16 @@ impl MarkdownWithToc<'_> {
crate fn into_string(self) -> String {
let MarkdownWithToc(md, mut ids, codes, edition, playground) = self;

let p = Parser::new_ext(md, opts());
let p = Parser::new_ext(md, opts()).into_offset_iter();

let mut s = String::with_capacity(md.len() * 3 / 2);

let mut toc = TocBuilder::new();

{
let p = HeadingLinks::new(p, Some(&mut toc), &mut ids);
let p = CodeBlocks::new(p, codes, edition, playground);
let p = Footnotes::new(p);
let p = CodeBlocks::new(p.map(|(ev, _)| ev), codes, edition, playground);
html::push_html(&mut s, p);
}

Expand All @@ -959,19 +963,19 @@ impl MarkdownHtml<'_> {
if md.is_empty() {
return String::new();
}
let p = Parser::new_ext(md, opts());
let p = Parser::new_ext(md, opts()).into_offset_iter();

// Treat inline HTML as plain text.
let p = p.map(|event| match event {
Event::Html(text) => Event::Text(text),
let p = p.map(|event| match event.0 {
Event::Html(text) => (Event::Text(text), event.1),
_ => event,
});

let mut s = String::with_capacity(md.len() * 3 / 2);

let p = HeadingLinks::new(p, None, &mut ids);
let p = CodeBlocks::new(p, codes, edition, playground);
let p = Footnotes::new(p);
let p = CodeBlocks::new(p.map(|(ev, _)| ev), codes, edition, playground);
html::push_html(&mut s, p);

s
Expand Down Expand Up @@ -1124,56 +1128,65 @@ crate fn plain_text_summary(md: &str) -> String {
s
}

crate fn markdown_links(md: &str) -> Vec<(String, Option<Range<usize>>)> {
crate fn markdown_links(md: &str) -> Vec<(String, Range<usize>)> {
if md.is_empty() {
return vec![];
}

let mut links = vec![];
let mut shortcut_links = vec![];

{
let locate = |s: &str| unsafe {
let s_start = s.as_ptr();
let s_end = s_start.add(s.len());
let md_start = md.as_ptr();
let md_end = md_start.add(md.len());
if md_start <= s_start && s_end <= md_end {
let start = s_start.offset_from(md_start) as usize;
let end = s_end.offset_from(md_start) as usize;
Some(start..end)
} else {
None
}
};
let links = RefCell::new(vec![]);

// FIXME: remove this function once pulldown_cmark can provide spans for link definitions.
let locate = |s: &str, fallback: Range<usize>| unsafe {
let s_start = s.as_ptr();
let s_end = s_start.add(s.len());
let md_start = md.as_ptr();
let md_end = md_start.add(md.len());
if md_start <= s_start && s_end <= md_end {
let start = s_start.offset_from(md_start) as usize;
let end = s_end.offset_from(md_start) as usize;
start..end
} else {
fallback
}
};

let span_for_link = |link: &CowStr<'_>, span: Range<usize>| {
// For diagnostics, we want to underline the link's definition but `span` will point at
// where the link is used. This is a problem for reference-style links, where the definition
// is separate from the usage.
match link {
// `Borrowed` variant means the string (the link's destination) may come directly from
// the markdown text and we can locate the original link destination.
// NOTE: LinkReplacer also provides `Borrowed` but possibly from other sources,
// so `locate()` can fall back to use `span`.
CowStr::Borrowed(s) => locate(s, span),

// For anything else, we can only use the provided range.
CowStr::Boxed(_) | CowStr::Inlined(_) => span,
}
};

let mut push = |link: BrokenLink<'_>| {
// FIXME: use `link.span` instead of `locate`
// (doing it now includes the `[]` as well as the text)
shortcut_links.push((link.reference.to_owned(), locate(link.reference)));
None
};
let p = Parser::new_with_broken_link_callback(md, opts(), Some(&mut push));

// There's no need to thread an IdMap through to here because
// the IDs generated aren't going to be emitted anywhere.
let mut ids = IdMap::new();
let iter = Footnotes::new(HeadingLinks::new(p, None, &mut ids));

for ev in iter {
if let Event::Start(Tag::Link(_, dest, _)) = ev {
debug!("found link: {}", dest);
links.push(match dest {
CowStr::Borrowed(s) => (s.to_owned(), locate(s)),
s @ (CowStr::Boxed(..) | CowStr::Inlined(..)) => (s.into_string(), None),
});
}
let mut push = |link: BrokenLink<'_>| {
let span = span_for_link(&CowStr::Borrowed(link.reference), link.span);
links.borrow_mut().push((link.reference.to_owned(), span));
None
};
let p = Parser::new_with_broken_link_callback(md, opts(), Some(&mut push)).into_offset_iter();

// There's no need to thread an IdMap through to here because
// the IDs generated aren't going to be emitted anywhere.
let mut ids = IdMap::new();
let iter = Footnotes::new(HeadingLinks::new(p, None, &mut ids));

for ev in iter {
if let Event::Start(Tag::Link(_, dest, _)) = ev.0 {
debug!("found link: {}", dest);
let span = span_for_link(&dest, ev.1);
links.borrow_mut().push((dest.into_string(), span));
}
}

links.append(&mut shortcut_links);

links
links.into_inner()
}

#[derive(Debug)]
Expand Down
Loading