diff options
author | Martin Fischer <martin@push-f.com> | 2021-04-08 08:42:01 +0200 |
---|---|---|
committer | Martin Fischer <martin@push-f.com> | 2021-04-08 15:40:37 +0200 |
commit | 57e7eefcbe6fb8c3dc4b01c707be9de4c34963a7 (patch) | |
tree | 6a9d296389bf3023396592c8514ed6712e011c7f /src |
import https://github.com/servo/html5ever
commit d1206daa740305f55a5fa159e43eb33afc359cb4
Diffstat (limited to 'src')
-rw-r--r-- | src/driver.rs | 137 | ||||
-rw-r--r-- | src/lib.rs | 30 | ||||
-rw-r--r-- | src/macros.rs | 33 | ||||
-rw-r--r-- | src/serialize/mod.rs | 256 | ||||
-rw-r--r-- | src/tokenizer/char_ref/mod.rs | 449 | ||||
-rw-r--r-- | src/tokenizer/interface.rs | 110 | ||||
-rw-r--r-- | src/tokenizer/mod.rs | 1713 | ||||
-rw-r--r-- | src/tokenizer/states.rs | 93 | ||||
-rw-r--r-- | src/tree_builder/data.rs | 171 | ||||
-rw-r--r-- | src/tree_builder/mod.rs | 1681 | ||||
-rw-r--r-- | src/tree_builder/rules.rs | 1449 | ||||
-rw-r--r-- | src/tree_builder/tag_sets.rs | 115 | ||||
-rw-r--r-- | src/tree_builder/types.rs | 95 | ||||
-rw-r--r-- | src/util/str.rs | 60 |
14 files changed, 6392 insertions, 0 deletions
diff --git a/src/driver.rs b/src/driver.rs new file mode 100644 index 0000000..26db9b8 --- /dev/null +++ b/src/driver.rs @@ -0,0 +1,137 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! High-level interface to the parser. + +use crate::buffer_queue::BufferQueue; +use crate::tokenizer::{Tokenizer, TokenizerOpts, TokenizerResult}; +use crate::tree_builder::{create_element, TreeBuilder, TreeBuilderOpts, TreeSink}; +use crate::{Attribute, QualName}; + +use std::borrow::Cow; + +use crate::tendril; +use crate::tendril::stream::{TendrilSink, Utf8LossyDecoder}; +use crate::tendril::StrTendril; + +/// All-encompassing options struct for the parser. +#[derive(Clone, Default)] +pub struct ParseOpts { + /// Tokenizer options. + pub tokenizer: TokenizerOpts, + + /// Tree builder options. + pub tree_builder: TreeBuilderOpts, +} + +/// Parse an HTML document +/// +/// The returned value implements `tendril::TendrilSink` +/// so that Unicode input may be provided incrementally, +/// or all at once with the `one` method. +/// +/// If your input is bytes, use `Parser::from_utf8`. +pub fn parse_document<Sink>(sink: Sink, opts: ParseOpts) -> Parser<Sink> +where + Sink: TreeSink, +{ + let tb = TreeBuilder::new(sink, opts.tree_builder); + let tok = Tokenizer::new(tb, opts.tokenizer); + Parser { + tokenizer: tok, + input_buffer: BufferQueue::new(), + } +} + +/// Parse an HTML fragment +/// +/// The returned value implements `tendril::TendrilSink` +/// so that Unicode input may be provided incrementally, +/// or all at once with the `one` method. +/// +/// If your input is bytes, use `Parser::from_utf8`. +pub fn parse_fragment<Sink>( + mut sink: Sink, + opts: ParseOpts, + context_name: QualName, + context_attrs: Vec<Attribute>, +) -> Parser<Sink> +where + Sink: TreeSink, +{ + let context_elem = create_element(&mut sink, context_name, context_attrs); + parse_fragment_for_element(sink, opts, context_elem, None) +} + +/// Like `parse_fragment`, but with an existing context element +/// and optionally a form element. +pub fn parse_fragment_for_element<Sink>( + sink: Sink, + opts: ParseOpts, + context_element: Sink::Handle, + form_element: Option<Sink::Handle>, +) -> Parser<Sink> +where + Sink: TreeSink, +{ + let tb = TreeBuilder::new_for_fragment(sink, context_element, form_element, opts.tree_builder); + let tok_opts = TokenizerOpts { + initial_state: Some(tb.tokenizer_state_for_context_elem()), + ..opts.tokenizer + }; + let tok = Tokenizer::new(tb, tok_opts); + Parser { + tokenizer: tok, + input_buffer: BufferQueue::new(), + } +} + +/// An HTML parser, +/// ready to receive Unicode input through the `tendril::TendrilSink` trait’s methods. +pub struct Parser<Sink> +where + Sink: TreeSink, +{ + pub tokenizer: Tokenizer<TreeBuilder<Sink::Handle, Sink>>, + pub input_buffer: BufferQueue, +} + +impl<Sink: TreeSink> TendrilSink<tendril::fmt::UTF8> for Parser<Sink> { + fn process(&mut self, t: StrTendril) { + self.input_buffer.push_back(t); + // FIXME: Properly support </script> somehow. + while let TokenizerResult::Script(_) = self.tokenizer.feed(&mut self.input_buffer) {} + } + + // FIXME: Is it too noisy to report every character decoding error? + fn error(&mut self, desc: Cow<'static, str>) { + self.tokenizer.sink.sink.parse_error(desc) + } + + type Output = Sink::Output; + + fn finish(mut self) -> Self::Output { + // FIXME: Properly support </script> somehow. + while let TokenizerResult::Script(_) = self.tokenizer.feed(&mut self.input_buffer) {} + assert!(self.input_buffer.is_empty()); + self.tokenizer.end(); + self.tokenizer.sink.sink.finish() + } +} + +impl<Sink: TreeSink> Parser<Sink> { + /// Wrap this parser into a `TendrilSink` that accepts UTF-8 bytes. + /// + /// Use this when your input is bytes that are known to be in the UTF-8 encoding. + /// Decoding is lossy, like `String::from_utf8_lossy`. + #[allow(clippy::wrong_self_convention)] + pub fn from_utf8(self) -> Utf8LossyDecoder<Self> { + Utf8LossyDecoder::new(self) + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..65fadaa --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,30 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![crate_name = "html5ever"] +#![crate_type = "dylib"] +#![cfg_attr(test, deny(warnings))] +#![allow(unused_parens)] + +pub use driver::{parse_document, parse_fragment, ParseOpts, Parser}; +pub use markup5ever::*; + +pub use serialize::serialize; + +#[macro_use] +mod macros; + +mod util { + pub mod str; +} + +pub mod driver; +pub mod serialize; +pub mod tokenizer; +pub mod tree_builder; diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 0000000..558a4a9 --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,33 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +macro_rules! unwrap_or_else { + ($opt:expr, $else_block:block) => { + match $opt { + None => $else_block, + Some(x) => x, + } + }; +} + +macro_rules! unwrap_or_return { + ($opt:expr, $retval:expr) => { + unwrap_or_else!($opt, { return $retval }) + }; +} + +macro_rules! time { + ($e:expr) => {{ + let now = ::std::time::Instant::now(); + let result = $e; + let d = now.elapsed(); + let dt = d.as_secs() * 1_000_000_000 + u64::from(d.subsec_nanos()); + (result, dt) + }}; +} diff --git a/src/serialize/mod.rs b/src/serialize/mod.rs new file mode 100644 index 0000000..3a57b47 --- /dev/null +++ b/src/serialize/mod.rs @@ -0,0 +1,256 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use log::warn; +pub use markup5ever::serialize::{AttrRef, Serialize, Serializer, TraversalScope}; +use markup5ever::{local_name, namespace_url, ns}; +use std::default::Default; +use std::io::{self, Write}; + +use crate::{LocalName, QualName}; + +pub fn serialize<Wr, T>(writer: Wr, node: &T, opts: SerializeOpts) -> io::Result<()> +where + Wr: Write, + T: Serialize, +{ + let mut ser = HtmlSerializer::new(writer, opts.clone()); + node.serialize(&mut ser, opts.traversal_scope) +} + +#[derive(Clone)] +pub struct SerializeOpts { + /// Is scripting enabled? + pub scripting_enabled: bool, + + /// Serialize the root node? Default: ChildrenOnly + pub traversal_scope: TraversalScope, + + /// If the serializer is asked to serialize an invalid tree, the default + /// behavior is to panic in the event that an `end_elem` is created without a + /// matching `start_elem`. Setting this to true will prevent those panics by + /// creating a default parent on the element stack. No extra start elem will + /// actually be written. Default: false + pub create_missing_parent: bool, +} + +impl Default for SerializeOpts { + fn default() -> SerializeOpts { + SerializeOpts { + scripting_enabled: true, + traversal_scope: TraversalScope::ChildrenOnly(None), + create_missing_parent: false, + } + } +} + +#[derive(Default)] +struct ElemInfo { + html_name: Option<LocalName>, + ignore_children: bool +} + +pub struct HtmlSerializer<Wr: Write> { + pub writer: Wr, + opts: SerializeOpts, + stack: Vec<ElemInfo>, +} + +fn tagname(name: &QualName) -> LocalName { + match name.ns { + ns!(html) | ns!(mathml) | ns!(svg) => (), + ref ns => { + // FIXME(#122) + warn!("node with weird namespace {:?}", ns); + }, + } + + name.local.clone() +} + +impl<Wr: Write> HtmlSerializer<Wr> { + pub fn new(writer: Wr, opts: SerializeOpts) -> Self { + let html_name = match opts.traversal_scope { + TraversalScope::IncludeNode | TraversalScope::ChildrenOnly(None) => None, + TraversalScope::ChildrenOnly(Some(ref n)) => Some(tagname(n)), + }; + HtmlSerializer { + writer, + opts, + stack: vec![ElemInfo { + html_name, + ignore_children: false, + }], + } + } + + fn parent(&mut self) -> &mut ElemInfo { + if self.stack.is_empty() { + if self.opts.create_missing_parent { + warn!("ElemInfo stack empty, creating new parent"); + self.stack.push(Default::default()); + } else { + panic!("no parent ElemInfo") + } + } + self.stack.last_mut().unwrap() + } + + fn write_escaped(&mut self, text: &str, attr_mode: bool) -> io::Result<()> { + for c in text.chars() { + match c { + '&' => self.writer.write_all(b"&"), + '\u{00A0}' => self.writer.write_all(b" "), + '"' if attr_mode => self.writer.write_all(b"""), + '<' if !attr_mode => self.writer.write_all(b"<"), + '>' if !attr_mode => self.writer.write_all(b">"), + c => self.writer.write_fmt(format_args!("{}", c)), + }?; + } + Ok(()) + } +} + +impl<Wr: Write> Serializer for HtmlSerializer<Wr> { + fn start_elem<'a, AttrIter>(&mut self, name: QualName, attrs: AttrIter) -> io::Result<()> + where + AttrIter: Iterator<Item = AttrRef<'a>>, + { + let html_name = match name.ns { + ns!(html) => Some(name.local.clone()), + _ => None, + }; + + if self.parent().ignore_children { + self.stack.push(ElemInfo { + html_name, + ignore_children: true, + }); + return Ok(()); + } + + self.writer.write_all(b"<")?; + self.writer.write_all(tagname(&name).as_bytes())?; + for (name, value) in attrs { + self.writer.write_all(b" ")?; + + match name.ns { + ns!() => (), + ns!(xml) => self.writer.write_all(b"xml:")?, + ns!(xmlns) => { + if name.local != local_name!("xmlns") { + self.writer.write_all(b"xmlns:")?; + } + }, + ns!(xlink) => self.writer.write_all(b"xlink:")?, + ref ns => { + // FIXME(#122) + warn!("attr with weird namespace {:?}", ns); + self.writer.write_all(b"unknown_namespace:")?; + }, + } + + self.writer.write_all(name.local.as_bytes())?; + self.writer.write_all(b"=\"")?; + self.write_escaped(value, true)?; + self.writer.write_all(b"\"")?; + } + self.writer.write_all(b">")?; + + let ignore_children = name.ns == ns!(html) && + match name.local { + local_name!("area") | + local_name!("base") | + local_name!("basefont") | + local_name!("bgsound") | + local_name!("br") | + local_name!("col") | + local_name!("embed") | + local_name!("frame") | + local_name!("hr") | + local_name!("img") | + local_name!("input") | + local_name!("keygen") | + local_name!("link") | + local_name!("meta") | + local_name!("param") | + local_name!("source") | + local_name!("track") | + local_name!("wbr") => true, + _ => false, + }; + + self.stack.push(ElemInfo { + html_name, + ignore_children, + }); + + Ok(()) + } + + fn end_elem(&mut self, name: QualName) -> io::Result<()> { + let info = match self.stack.pop() { + Some(info) => info, + None if self.opts.create_missing_parent => { + warn!("missing ElemInfo, creating default."); + Default::default() + }, + _ => panic!("no ElemInfo"), + }; + if info.ignore_children { + return Ok(()); + } + + self.writer.write_all(b"</")?; + self.writer.write_all(tagname(&name).as_bytes())?; + self.writer.write_all(b">") + } + + fn write_text(&mut self, text: &str) -> io::Result<()> { + let escape = match self.parent().html_name { + Some(local_name!("style")) | + Some(local_name!("script")) | + Some(local_name!("xmp")) | + Some(local_name!("iframe")) | + Some(local_name!("noembed")) | + Some(local_name!("noframes")) | + Some(local_name!("plaintext")) => false, + + Some(local_name!("noscript")) => !self.opts.scripting_enabled, + + _ => true, + }; + + if escape { + self.write_escaped(text, false) + } else { + self.writer.write_all(text.as_bytes()) + } + } + + fn write_comment(&mut self, text: &str) -> io::Result<()> { + self.writer.write_all(b"<!--")?; + self.writer.write_all(text.as_bytes())?; + self.writer.write_all(b"-->") + } + + fn write_doctype(&mut self, name: &str) -> io::Result<()> { + self.writer.write_all(b"<!DOCTYPE ")?; + self.writer.write_all(name.as_bytes())?; + self.writer.write_all(b">") + } + + fn write_processing_instruction(&mut self, target: &str, data: &str) -> io::Result<()> { + self.writer.write_all(b"<?")?; + self.writer.write_all(target.as_bytes())?; + self.writer.write_all(b" ")?; + self.writer.write_all(data.as_bytes())?; + self.writer.write_all(b">") + } +} diff --git a/src/tokenizer/char_ref/mod.rs b/src/tokenizer/char_ref/mod.rs new file mode 100644 index 0000000..a52485d --- /dev/null +++ b/src/tokenizer/char_ref/mod.rs @@ -0,0 +1,449 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use super::{TokenSink, Tokenizer}; +use crate::buffer_queue::BufferQueue; +use crate::data; +use crate::tendril::StrTendril; +use crate::util::str::is_ascii_alnum; + +use log::debug; +use mac::format_if; +use std::borrow::Cow::Borrowed; +use std::char::from_u32; + +use self::State::*; +pub use self::Status::*; + +//§ tokenizing-character-references +pub struct CharRef { + /// The resulting character(s) + pub chars: [char; 2], + + /// How many slots in `chars` are valid? + pub num_chars: u8, +} + +pub enum Status { + Stuck, + Progress, + Done, +} + +#[derive(Debug)] +enum State { + Begin, + Octothorpe, + Numeric(u32), // base + NumericSemicolon, + Named, + BogusName, +} + +pub struct CharRefTokenizer { + state: State, + addnl_allowed: Option<char>, + result: Option<CharRef>, + + num: u32, + num_too_big: bool, + seen_digit: bool, + hex_marker: Option<char>, + + name_buf_opt: Option<StrTendril>, + name_match: Option<(u32, u32)>, + name_len: usize, +} + +impl CharRefTokenizer { + // NB: We assume that we have an additional allowed character iff we're + // tokenizing in an attribute value. + pub fn new(addnl_allowed: Option<char>) -> CharRefTokenizer { + CharRefTokenizer { + state: Begin, + addnl_allowed, + result: None, + num: 0, + num_too_big: false, + seen_digit: false, + hex_marker: None, + name_buf_opt: None, + name_match: None, + name_len: 0, + } + } + + // A CharRefTokenizer can only tokenize one character reference, + // so this method consumes the tokenizer. + pub fn get_result(self) -> CharRef { + self.result.expect("get_result called before done") + } + + fn name_buf(&self) -> &StrTendril { + self.name_buf_opt + .as_ref() + .expect("name_buf missing in named character reference") + } + + fn name_buf_mut(&mut self) -> &mut StrTendril { + self.name_buf_opt + .as_mut() + .expect("name_buf missing in named character reference") + } + + fn finish_none(&mut self) -> Status { + self.result = Some(CharRef { + chars: ['\0', '\0'], + num_chars: 0, + }); + Done + } + + fn finish_one(&mut self, c: char) -> Status { + self.result = Some(CharRef { + chars: [c, '\0'], + num_chars: 1, + }); + Done + } +} + +impl CharRefTokenizer { + pub fn step<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) -> Status { + if self.result.is_some() { + return Done; + } + + debug!("char ref tokenizer stepping in state {:?}", self.state); + match self.state { + Begin => self.do_begin(tokenizer, input), + Octothorpe => self.do_octothorpe(tokenizer, input), + Numeric(base) => self.do_numeric(tokenizer, input, base), + NumericSemicolon => self.do_numeric_semicolon(tokenizer, input), + Named => self.do_named(tokenizer, input), + BogusName => self.do_bogus_name(tokenizer, input), + } + } + + fn do_begin<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) -> Status { + match unwrap_or_return!(tokenizer.peek(input), Stuck) { + '\t' | '\n' | '\x0C' | ' ' | '<' | '&' => self.finish_none(), + c if Some(c) == self.addnl_allowed => self.finish_none(), + + '#' => { + tokenizer.discard_char(input); + self.state = Octothorpe; + Progress + }, + + _ => { + self.state = Named; + self.name_buf_opt = Some(StrTendril::new()); + Progress + }, + } + } + + fn do_octothorpe<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) -> Status { + let c = unwrap_or_return!(tokenizer.peek(input), Stuck); + match c { + 'x' | 'X' => { + tokenizer.discard_char(input); + self.hex_marker = Some(c); + self.state = Numeric(16); + }, + + _ => { + self.hex_marker = None; + self.state = Numeric(10); + }, + } + Progress + } + + fn do_numeric<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + base: u32, + ) -> Status { + let c = unwrap_or_return!(tokenizer.peek(input), Stuck); + match c.to_digit(base) { + Some(n) => { + tokenizer.discard_char(input); + self.num = self.num.wrapping_mul(base); + if self.num > 0x10FFFF { + // We might overflow, and the character is definitely invalid. + // We still parse digits and semicolon, but don't use the result. + self.num_too_big = true; + } + self.num = self.num.wrapping_add(n); + self.seen_digit = true; + Progress + }, + + None if !self.seen_digit => self.unconsume_numeric(tokenizer, input), + + None => { + self.state = NumericSemicolon; + Progress + }, + } + } + + fn do_numeric_semicolon<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) -> Status { + match unwrap_or_return!(tokenizer.peek(input), Stuck) { + ';' => tokenizer.discard_char(input), + _ => tokenizer.emit_error(Borrowed( + "Semicolon missing after numeric character reference", + )), + }; + self.finish_numeric(tokenizer) + } + + fn unconsume_numeric<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) -> Status { + let mut unconsume = StrTendril::from_char('#'); + match self.hex_marker { + Some(c) => unconsume.push_char(c), + None => (), + } + + input.push_front(unconsume); + tokenizer.emit_error(Borrowed("Numeric character reference without digits")); + self.finish_none() + } + + fn finish_numeric<Sink: TokenSink>(&mut self, tokenizer: &mut Tokenizer<Sink>) -> Status { + fn conv(n: u32) -> char { + from_u32(n).expect("invalid char missed by error handling cases") + } + + let (c, error) = match self.num { + n if (n > 0x10FFFF) || self.num_too_big => ('\u{fffd}', true), + 0x00 | 0xD800..=0xDFFF => ('\u{fffd}', true), + + 0x80..=0x9F => match data::C1_REPLACEMENTS[(self.num - 0x80) as usize] { + Some(c) => (c, true), + None => (conv(self.num), true), + }, + + 0x01..=0x08 | 0x0B | 0x0D..=0x1F | 0x7F | 0xFDD0..=0xFDEF => (conv(self.num), true), + + n if (n & 0xFFFE) == 0xFFFE => (conv(n), true), + + n => (conv(n), false), + }; + + if error { + let msg = format_if!( + tokenizer.opts.exact_errors, + "Invalid numeric character reference", + "Invalid numeric character reference value 0x{:06X}", + self.num + ); + tokenizer.emit_error(msg); + } + + self.finish_one(c) + } + + fn do_named<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) -> Status { + let c = unwrap_or_return!(tokenizer.get_char(input), Stuck); + self.name_buf_mut().push_char(c); + match data::NAMED_ENTITIES.get(&self.name_buf()[..]) { + // We have either a full match or a prefix of one. + Some(&m) => { + if m.0 != 0 { + // We have a full match, but there might be a longer one to come. + self.name_match = Some(m); + self.name_len = self.name_buf().len(); + } + // Otherwise we just have a prefix match. + Progress + }, + + // Can't continue the match. + None => self.finish_named(tokenizer, input, Some(c)), + } + } + + fn emit_name_error<Sink: TokenSink>(&mut self, tokenizer: &mut Tokenizer<Sink>) { + let msg = format_if!( + tokenizer.opts.exact_errors, + "Invalid character reference", + "Invalid character reference &{}", + self.name_buf() + ); + tokenizer.emit_error(msg); + } + + fn unconsume_name(&mut self, input: &mut BufferQueue) { + input.push_front(self.name_buf_opt.take().unwrap()); + } + + fn finish_named<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + end_char: Option<char>, + ) -> Status { + match self.name_match { + None => { + match end_char { + Some(c) if is_ascii_alnum(c) => { + // Keep looking for a semicolon, to determine whether + // we emit a parse error. + self.state = BogusName; + return Progress; + }, + + // Check length because &; is not a parse error. + Some(';') if self.name_buf().len() > 1 => self.emit_name_error(tokenizer), + + _ => (), + } + self.unconsume_name(input); + self.finish_none() + }, + + Some((c1, c2)) => { + // We have a complete match, but we may have consumed + // additional characters into self.name_buf. Usually + // at least one, but several in cases like + // + // ¬ => match for U+00AC + // ¬i => valid prefix for ¬in + // ¬it => can't continue match + + let name_len = self.name_len; + assert!(name_len > 0); + let last_matched = self.name_buf()[name_len - 1..].chars().next().unwrap(); + + // There might not be a next character after the match, if + // we had a full match and then hit EOF. + let next_after = if name_len == self.name_buf().len() { + None + } else { + Some(self.name_buf()[name_len..].chars().next().unwrap()) + }; + + // "If the character reference is being consumed as part of an + // attribute, and the last character matched is not a U+003B + // SEMICOLON character (;), and the next character is either a + // U+003D EQUALS SIGN character (=) or an alphanumeric ASCII + // character, then, for historical reasons, all the characters + // that were matched after the U+0026 AMPERSAND character (&) + // must be unconsumed, and nothing is returned. However, if + // this next character is in fact a U+003D EQUALS SIGN + // character (=), then this is a parse error" + + let unconsume_all = match (self.addnl_allowed, last_matched, next_after) { + (_, ';', _) => false, + (Some(_), _, Some('=')) => { + tokenizer.emit_error(Borrowed( + "Equals sign after character reference in attribute", + )); + true + }, + (Some(_), _, Some(c)) if is_ascii_alnum(c) => true, + _ => { + tokenizer.emit_error(Borrowed( + "Character reference does not end with semicolon", + )); + false + }, + }; + + if unconsume_all { + self.unconsume_name(input); + self.finish_none() + } else { + input.push_front(StrTendril::from_slice(&self.name_buf()[name_len..])); + self.result = Some(CharRef { + chars: [from_u32(c1).unwrap(), from_u32(c2).unwrap()], + num_chars: if c2 == 0 { 1 } else { 2 }, + }); + Done + } + }, + } + } + + fn do_bogus_name<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) -> Status { + let c = unwrap_or_return!(tokenizer.get_char(input), Stuck); + self.name_buf_mut().push_char(c); + match c { + _ if is_ascii_alnum(c) => return Progress, + ';' => self.emit_name_error(tokenizer), + _ => (), + } + self.unconsume_name(input); + self.finish_none() + } + + pub fn end_of_file<Sink: TokenSink>( + &mut self, + tokenizer: &mut Tokenizer<Sink>, + input: &mut BufferQueue, + ) { + while self.result.is_none() { + match self.state { + Begin => drop(self.finish_none()), + + Numeric(_) if !self.seen_digit => drop(self.unconsume_numeric(tokenizer, input)), + + Numeric(_) | NumericSemicolon => { + tokenizer.emit_error(Borrowed("EOF in numeric character reference")); + self.finish_numeric(tokenizer); + }, + + Named => drop(self.finish_named(tokenizer, input, None)), + + BogusName => { + self.unconsume_name(input); + self.finish_none(); + }, + + Octothorpe => { + input.push_front(StrTendril::from_slice("#")); + tokenizer.emit_error(Borrowed("EOF after '#' in character reference")); + self.finish_none(); + }, + } + } + } +} diff --git a/src/tokenizer/interface.rs b/src/tokenizer/interface.rs new file mode 100644 index 0000000..22d11be --- /dev/null +++ b/src/tokenizer/interface.rs @@ -0,0 +1,110 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use crate::interface::Attribute; +use crate::tendril::StrTendril; +use crate::tokenizer::states; +use crate::LocalName; +use std::borrow::Cow; + +pub use self::TagKind::{EndTag, StartTag}; +pub use self::Token::{CharacterTokens, CommentToken, DoctypeToken, TagToken}; +pub use self::Token::{EOFToken, NullCharacterToken, ParseError}; + +/// A `DOCTYPE` token. +// FIXME: already exists in Servo DOM +#[derive(PartialEq, Eq, Clone, Debug)] +pub struct Doctype { + pub name: Option<StrTendril>, + pub public_id: Option<StrTendril>, + pub system_id: Option<StrTendril>, + pub force_quirks: bool, +} + +impl Doctype { + pub fn new() -> Doctype { + Doctype { + name: None, + public_id: None, + system_id: None, + force_quirks: false, + } + } +} + +#[derive(PartialEq, Eq, Hash, Copy, Clone, Debug)] +pub enum TagKind { + StartTag, + EndTag, +} + +/// A tag token. +#[derive(PartialEq, Eq, Clone, Debug)] +pub struct Tag { + pub kind: TagKind, + pub name: LocalName, + pub self_closing: bool, + pub attrs: Vec<Attribute>, +} + +impl Tag { + /// Are the tags equivalent when we don't care about attribute order? + /// Also ignores the self-closing flag. + pub fn equiv_modulo_attr_order(&self, other: &Tag) -> bool { + if (self.kind != other.kind) || (self.name != other.name) { + return false; + } + + let mut self_attrs = self.attrs.clone(); + let mut other_attrs = other.attrs.clone(); + self_attrs.sort(); + other_attrs.sort(); + + self_attrs == other_attrs + } +} + +#[derive(PartialEq, Eq, Debug)] +pub enum Token { + DoctypeToken(Doctype), + TagToken(Tag), + CommentToken(StrTendril), + CharacterTokens(StrTendril), + NullCharacterToken, + EOFToken, + ParseError(Cow<'static, str>), +} + +#[derive(Debug, PartialEq)] +#[must_use] +pub enum TokenSinkResult<Handle> { + Continue, + Script(Handle), + Plaintext, + RawData(states::RawKind), +} + +/// Types which can receive tokens from the tokenizer. +pub trait TokenSink { + type Handle; + + /// Process a token. + fn process_token(&mut self, token: Token, line_number: u64) -> TokenSinkResult<Self::Handle>; + + // Signal sink that tokenization reached the end. + fn end(&mut self) {} + + /// Used in the markup declaration open state. By default, this always + /// returns false and thus all CDATA sections are tokenized as bogus + /// comments. + /// https://html.spec.whatwg.org/multipage/#markup-declaration-open-state + fn adjusted_current_node_present_but_not_in_html_namespace(&self) -> bool { + false + } +} diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs new file mode 100644 index 0000000..267fdf3 --- /dev/null +++ b/src/tokenizer/mod.rs @@ -0,0 +1,1713 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! The HTML5 tokenizer. + +pub use self::interface::{CharacterTokens, EOFToken, NullCharacterToken, ParseError}; +pub use self::interface::{CommentToken, DoctypeToken, TagToken, Token}; +pub use self::interface::{Doctype, EndTag, StartTag, Tag, TagKind}; +pub use self::interface::{TokenSink, TokenSinkResult}; + +use self::states::{DoctypeIdKind, Public, System}; +use self::states::{DoubleEscaped, Escaped}; +use self::states::{DoubleQuoted, SingleQuoted, Unquoted}; +use self::states::{Rawtext, Rcdata, ScriptData, ScriptDataEscaped}; + +use self::char_ref::{CharRef, CharRefTokenizer}; + +use crate::util::str::lower_ascii_letter; + +use log::debug; +use mac::{_tt_as_expr_hack, format_if, matches}; +use markup5ever::{namespace_url, ns, small_char_set}; +use std::borrow::Cow::{self, Borrowed}; +use std::collections::BTreeMap; +use std::default::Default; +use std::mem::replace; + +pub use crate::buffer_queue::{BufferQueue, FromSet, NotFromSet, SetResult}; +use crate::tendril::StrTendril; +use crate::{Attribute, LocalName, QualName, SmallCharSet}; + +mod char_ref; +mod interface; +pub mod states; + +pub enum ProcessResult<Handle> { + Continue, + Suspend, + Script(Handle), +} + +#[must_use] +pub enum TokenizerResult<Handle> { + Done, + Script(Handle), +} + +fn option_push(opt_str: &mut Option<StrTendril>, c: char) { + match *opt_str { + Some(ref mut s) => s.push_char(c), + None => *opt_str = Some(StrTendril::from_char(c)), + } +} + +/// Tokenizer options, with an impl for `Default`. +#[derive(Clone)] +pub struct TokenizerOpts { + /// Report all parse errors described in the spec, at some + /// performance penalty? Default: false + pub exact_errors: bool, + + /// Discard a `U+FEFF BYTE ORDER MARK` if we see one at the beginning + /// of the stream? Default: true + pub discard_bom: bool, + + /// Keep a record of how long we spent in each state? Printed + /// when `end()` is called. Default: false + pub profile: bool, + + /// Initial state override. Only the test runner should use + /// a non-`None` value! + pub initial_state: Option<states::State>, + + /// Last start tag. Only the test runner should use a + /// non-`None` value! + /// + /// FIXME: Can't use Tendril because we want TokenizerOpts + /// to be Send. + pub last_start_tag_name: Option<String>, +} + +impl Default for TokenizerOpts { + fn default() -> TokenizerOpts { + TokenizerOpts { + exact_errors: false, + discard_bom: true, + profile: false, + initial_state: None, + last_start_tag_name: None, + } + } +} + +/// The HTML tokenizer. +pub struct Tokenizer<Sink> { + /// Options controlling the behavior of the tokenizer. + opts: TokenizerOpts, + + /// Destination for tokens we emit. + pub sink: Sink, + + /// The abstract machine state as described in the spec. + state: states::State, + + /// Are we at the end of the file, once buffers have been processed + /// completely? This affects whether we will wait for lookahead or not. + at_eof: bool, + + /// Tokenizer for character references, if we're tokenizing + /// one at the moment. + char_ref_tokenizer: Option<Box<CharRefTokenizer>>, + + /// Current input character. Just consumed, may reconsume. + current_char: char, + + /// Should we reconsume the current input character? + reconsume: bool, + + /// Did we just consume \r, translating it to \n? In that case we need + /// to ignore the next character if it's \n. + ignore_lf: bool, + + /// Discard a U+FEFF BYTE ORDER MARK if we see one? Only done at the + /// beginning of the stream. + discard_bom: bool, + + /// Current tag kind. + current_tag_kind: TagKind, + + /// Current tag name. + current_tag_name: StrTendril, + + /// Current tag is self-closing? + current_tag_self_closing: bool, + + /// Current tag attributes. + current_tag_attrs: Vec<Attribute>, + + /// Current attribute name. + current_attr_name: StrTendril, + + /// Current attribute value. + current_attr_value: StrTendril, + + /// Current comment. + current_comment: StrTendril, + + /// Current doctype token. + current_doctype: Doctype, + + /// Last start tag name, for use in checking "appropriate end tag". + last_start_tag_name: Option<LocalName>, + + /// The "temporary buffer" mentioned in the spec. + temp_buf: StrTendril, + + /// Record of how many ns we spent in each state, if profiling is enabled. + state_profile: BTreeMap<states::State, u64>, + + /// Record of how many ns we spent in the token sink. + time_in_sink: u64, + + /// Track current line + current_line: u64, +} + +impl<Sink: TokenSink> Tokenizer<Sink> { + /// Create a new tokenizer which feeds tokens to a particular `TokenSink`. + pub fn new(sink: Sink, mut opts: TokenizerOpts) -> Tokenizer<Sink> { + let start_tag_name = opts + .last_start_tag_name + .take() + .map(|s| LocalName::from(&*s)); + let state = opts.initial_state.unwrap_or(states::Data); + let discard_bom = opts.discard_bom; + Tokenizer { + opts, + sink, + state, + char_ref_tokenizer: None, + at_eof: false, + current_char: '\0', + reconsume: false, + ignore_lf: false, + discard_bom, + current_tag_kind: StartTag, + current_tag_name: StrTendril::new(), + current_tag_self_closing: false, + current_tag_attrs: vec![], + current_attr_name: StrTendril::new(), + current_attr_value: StrTendril::new(), + current_comment: StrTendril::new(), + current_doctype: Doctype::new(), + last_start_tag_name: start_tag_name, + temp_buf: StrTendril::new(), + state_profile: BTreeMap::new(), + time_in_sink: 0, + current_line: 1, + } + } + + /// Feed an input string into the tokenizer. + pub fn feed(&mut self, input: &mut BufferQueue) -> TokenizerResult<Sink::Handle> { + if input.is_empty() { + return TokenizerResult::Done; + } + + if self.discard_bom { + if let Some(c) = input.peek() { + if c == '\u{feff}' { + input.next(); + } + } else { + return TokenizerResult::Done; + } + }; + + self.run(input) + } + + pub fn set_plaintext_state(&mut self) { + self.state = states::Plaintext; + } + + fn process_token(&mut self, token: Token) -> TokenSinkResult<Sink::Handle> { + if self.opts.profile { + let (ret, dt) = time!(self.sink.process_token(token, self.current_line)); + self.time_in_sink += dt; + ret + } else { + self.sink.process_token(token, self.current_line) + } + } + + fn process_token_and_continue(&mut self, token: Token) { + assert!(matches!( + self.process_token(token), + TokenSinkResult::Continue + )); + } + + //§ preprocessing-the-input-stream + // Get the next input character, which might be the character + // 'c' that we already consumed from the buffers. + fn get_preprocessed_char(&mut self, mut c: char, input: &mut BufferQueue) -> Option<char> { + if self.ignore_lf { + self.ignore_lf = false; + if c == '\n' { + c = unwrap_or_return!(input.next(), None); + } + } + + if c == '\r' { + self.ignore_lf = true; + c = '\n'; + } + + if c == '\n' { + self.current_line += 1; + } + + if self.opts.exact_errors && + match c as u32 { + 0x01..=0x08 | 0x0B | 0x0E..=0x1F | 0x7F..=0x9F | 0xFDD0..=0xFDEF => true, + n if (n & 0xFFFE) == 0xFFFE => true, + _ => false, + } + { + let msg = format!("Bad character {}", c); + self.emit_error(Cow::Owned(msg)); + } + + debug!("got character {}", c); + self.current_char = c; + Some(c) + } + + //§ tokenization + // Get the next input character, if one is available. + fn get_char(&mut self, input: &mut BufferQueue) -> Option<char> { + if self.reconsume { + self.reconsume = false; + Some(self.current_char) + } else { + input + .next() + .and_then(|c| self.get_preprocessed_char(c, input)) + } + } + + fn pop_except_from(&mut self, input: &mut BufferQueue, set: SmallCharSet) -> Option<SetResult> { + // Bail to the slow path for various corner cases. + // This means that `FromSet` can contain characters not in the set! + // It shouldn't matter because the fallback `FromSet` case should + // always do the same thing as the `NotFromSet` case. + if self.opts.exact_errors || self.reconsume || self.ignore_lf { + return self.get_char(input).map(FromSet); + } + + let d = input.pop_except_from(set); + debug!("got characters {:?}", d); + match d { + Some(FromSet(c)) => self.get_preprocessed_char(c, input).map(FromSet), + + // NB: We don't set self.current_char for a run of characters not + // in the set. It shouldn't matter for the codepaths that use + // this. + _ => d, + } + } + + // Check if the next characters are an ASCII case-insensitive match. See + // BufferQueue::eat. + // + // NB: this doesn't do input stream preprocessing or set the current input + // character. + fn eat( + &mut self, + input: &mut BufferQueue, + pat: &str, + eq: fn(&u8, &u8) -> bool, + ) -> Option<bool> { + input.push_front(replace(&mut self.temp_buf, StrTendril::new())); + match input.eat(pat, eq) { + None if self.at_eof => Some(false), + None => { + while let Some(c) = input.next() { + self.temp_buf.push_char(c); + } + None + }, + Some(matched) => Some(matched), + } + } + + /// Run the state machine for as long as we can. + fn run(&mut self, input: &mut BufferQueue) -> TokenizerResult<Sink::Handle> { + if self.opts.profile { + loop { + let state = self.state; + let old_sink = self.time_in_sink; + let (run, mut dt) = time!(self.step(input)); + dt -= (self.time_in_sink - old_sink); + let new = match self.state_profile.get_mut(&state) { + Some(x) => { + *x += dt; + false + }, + None => true, + }; + if new { + // do this here because of borrow shenanigans + self.state_profile.insert(state, dt); + } + match run { + ProcessResult::Continue => (), + ProcessResult::Suspend => break, + ProcessResult::Script(node) => return TokenizerResult::Script(node), + } + } + } else { + loop { + match self.step(input) { + ProcessResult::Continue => (), + ProcessResult::Suspend => break, + ProcessResult::Script(node) => return TokenizerResult::Script(node), + } + } + } + TokenizerResult::Done + } + + fn bad_char_error(&mut self) { + let msg = format_if!( + self.opts.exact_errors, + "Bad character", + "Saw {} in state {:?}", + self.current_char, + self.state + ); + self.emit_error(msg); + } + + fn bad_eof_error(&mut self) { + let msg = format_if!( + self.opts.exact_errors, + "Unexpected EOF", + "Saw EOF in state {:?}", + self.state + ); + self.emit_error(msg); + } + + fn emit_char(&mut self, c: char) { + self.process_token_and_continue(match c { + '\0' => NullCharacterToken, + _ => CharacterTokens(StrTendril::from_char(c)), + }); + } + + // The string must not contain '\0'! + fn emit_chars(&mut self, b: StrTendril) { + self.process_token_and_continue(CharacterTokens(b)); + } + + fn emit_current_tag(&mut self) -> ProcessResult<Sink::Handle> { + self.finish_attribute(); + + let name = LocalName::from(&*self.current_tag_name); + self.current_tag_name.clear(); + + match self.current_tag_kind { + StartTag => { + self.last_start_tag_name = Some(name.clone()); + }, + EndTag => { + if !self.current_tag_attrs.is_empty() { + self.emit_error(Borrowed("Attributes on an end tag")); + } + if self.current_tag_self_closing { + self.emit_error(Borrowed("Self-closing end tag")); + } + }, + } + + let token = TagToken(Tag { + kind: self.current_tag_kind, + name, + self_closing: self.current_tag_self_closing, + attrs: replace(&mut self.current_tag_attrs, vec![]), + }); + + match self.process_token(token) { + TokenSinkResult::Continue => ProcessResult::Continue, + TokenSinkResult::Plaintext => { + self.state = states::Plaintext; + ProcessResult::Continue + }, + TokenSinkResult::Script(node) => { + self.state = states::Data; + ProcessResult::Script(node) + }, + TokenSinkResult::RawData(kind) => { + self.state = states::RawData(kind); + ProcessResult::Continue + }, + } + } + + fn emit_temp_buf(&mut self) { + // FIXME: Make sure that clearing on emit is spec-compatible. + let buf = replace(&mut self.temp_buf, StrTendril::new()); + self.emit_chars(buf); + } + + fn clear_temp_buf(&mut self) { + // Do this without a new allocation. + self.temp_buf.clear(); + } + + fn emit_current_comment(&mut self) { + let comment = replace(&mut self.current_comment, StrTendril::new()); + self.process_token_and_continue(CommentToken(comment)); + } + + fn discard_tag(&mut self) { + self.current_tag_name.clear(); + self.current_tag_self_closing = false; + self.current_tag_attrs = vec![]; + } + + fn create_tag(&mut self, kind: TagKind, c: char) { + self.discard_tag(); + self.current_tag_name.push_char(c); + self.current_tag_kind = kind; + } + + fn have_appropriate_end_tag(&self) -> bool { + match self.last_start_tag_name.as_ref() { + Some(last) => (self.current_tag_kind == EndTag) && (*self.current_tag_name == **last), + None => false, + } + } + + fn create_attribute(&mut self, c: char) { + self.finish_attribute(); + + self.current_attr_name.push_char(c); + } + + fn finish_attribute(&mut self) { + if self.current_attr_name.is_empty() { + return; + } + + // Check for a duplicate attribute. + // FIXME: the spec says we should error as soon as the name is finished. + // FIXME: linear time search, do we care? + let dup = { + let name = &*self.current_attr_name; + self.current_tag_attrs + .iter() + .any(|a| &*a.name.local == name) + }; + + if dup { + self.emit_error(Borrowed("Duplicate attribute")); + self.current_attr_name.clear(); + self.current_attr_value.clear(); + } else { + let name = LocalName::from(&*self.current_attr_name); + self.current_attr_name.clear(); + self.current_tag_attrs.push(Attribute { + // The tree builder will adjust the namespace if necessary. + // This only happens in foreign elements. + name: QualName::new(None, ns!(), name), + value: replace(&mut self.current_attr_value, StrTendril::new()), + }); + } + } + + fn emit_current_doctype(&mut self) { + let doctype = replace(&mut self.current_doctype, Doctype::new()); + self.process_token_and_continue(DoctypeToken(doctype)); + } + + fn doctype_id(&mut self, kind: DoctypeIdKind) -> &mut Option<StrTendril> { + match kind { + Public => &mut self.current_doctype.public_id, + System => &mut self.current_doctype.system_id, + } + } + + fn clear_doctype_id(&mut self, kind: DoctypeIdKind) { + let id = self.doctype_id(kind); + match *id { + Some(ref mut s) => s.clear(), + None => *id = Some(StrTendril::new()), + } + } + + fn consume_char_ref(&mut self, addnl_allowed: Option<char>) { + // NB: The char ref tokenizer assumes we have an additional allowed + // character iff we're tokenizing in an attribute value. + self.char_ref_tokenizer = Some(Box::new(CharRefTokenizer::new(addnl_allowed))); + } + + fn emit_eof(&mut self) { + self.process_token_and_continue(EOFToken); + } + + fn peek(&mut self, input: &BufferQueue) -> Option<char> { + if self.reconsume { + Some(self.current_char) + } else { + input.peek() + } + } + + fn discard_char(&mut self, input: &mut BufferQueue) { + self.get_char(input); + } + + fn emit_error(&mut self, error: Cow<'static, str>) { + self.process_token_and_continue(ParseError(error)); + } +} +//§ END + +// Shorthand for common state machine behaviors. +macro_rules! shorthand ( + ( $me:ident : emit $c:expr ) => ( $me.emit_char($c); ); + ( $me:ident : create_tag $kind:ident $c:expr ) => ( $me.create_tag($kind, $c); ); + ( $me:ident : push_tag $c:expr ) => ( $me.current_tag_name.push_char($c); ); + ( $me:ident : discard_tag ) => ( $me.discard_tag(); ); + ( $me:ident : discard_char $input:expr ) => ( $me.discard_char($input); ); + ( $me:ident : push_temp $c:expr ) => ( $me.temp_buf.push_char($c); ); + ( $me:ident : emit_temp ) => ( $me.emit_temp_buf(); ); + ( $me:ident : clear_temp ) => ( $me.clear_temp_buf(); ); + ( $me:ident : create_attr $c:expr ) => ( $me.create_attribute($c); ); + ( $me:ident : push_name $c:expr ) => ( $me.current_attr_name.push_char($c); ); + ( $me:ident : push_value $c:expr ) => ( $me.current_attr_value.push_char($c); ); + ( $me:ident : append_value $c:expr ) => ( $me.current_attr_value.push_tendril($c); ); + ( $me:ident : push_comment $c:expr ) => ( $me.current_comment.push_char($c); ); + ( $me:ident : append_comment $c:expr ) => ( $me.current_comment.push_slice($c); ); + ( $me:ident : emit_comment ) => ( $me.emit_current_comment(); ); + ( $me:ident : clear_comment ) => ( $me.current_comment.clear(); ); + ( $me:ident : create_doctype ) => ( $me.current_doctype = Doctype::new(); ); + ( $me:ident : push_doctype_name $c:expr ) => ( option_push(&mut $me.current_doctype.name, $c); ); + ( $me:ident : push_doctype_id $k:ident $c:expr ) => ( option_push($me.doctype_id($k), $c); ); + ( $me:ident : clear_doctype_id $k:ident ) => ( $me.clear_doctype_id($k); ); + ( $me:ident : force_quirks ) => ( $me.current_doctype.force_quirks = true; ); + ( $me:ident : emit_doctype ) => ( $me.emit_current_doctype(); ); + ( $me:ident : error ) => ( $me.bad_char_error(); ); + ( $me:ident : error_eof ) => ( $me.bad_eof_error(); ); +); + +// Tracing of tokenizer actions. This adds significant bloat and compile time, +// so it's behind a cfg flag. +#[cfg(trace_tokenizer)] +macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ({ + debug!(" {:s}", stringify!($($cmds)*)); + shorthand!($me:expr : $($cmds)*); +})); + +#[cfg(not(trace_tokenizer))] +macro_rules! sh_trace ( ( $me:ident : $($cmds:tt)* ) => ( shorthand!($me: $($cmds)*) ) ); + +// A little DSL for sequencing shorthand actions. +macro_rules! go ( + // A pattern like $($cmd:tt)* ; $($rest:tt)* causes parse ambiguity. + // We have to tell the parser how much lookahead we need. + + ( $me:ident : $a:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a); go!($me: $($rest)*); }); + ( $me:ident : $a:tt $b:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b); go!($me: $($rest)*); }); + ( $me:ident : $a:tt $b:tt $c:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c); go!($me: $($rest)*); }); + ( $me:ident : $a:tt $b:tt $c:tt $d:tt ; $($rest:tt)* ) => ({ sh_trace!($me: $a $b $c $d); go!($me: $($rest)*); }); + + // These can only come at the end. + + ( $me:ident : to $s:ident ) => ({ $me.state = states::$s; return ProcessResult::Continue; }); + ( $me:ident : to $s:ident $k1:expr ) => ({ $me.state = states::$s($k1); return ProcessResult::Continue; }); + ( $me:ident : to $s:ident $k1:ident $k2:expr ) => ({ $me.state = states::$s($k1($k2)); return ProcessResult::Continue; }); + + ( $me:ident : reconsume $s:ident ) => ({ $me.reconsume = true; go!($me: to $s); }); + ( $me:ident : reconsume $s:ident $k1:expr ) => ({ $me.reconsume = true; go!($me: to $s $k1); }); + ( $me:ident : reconsume $s:ident $k1:ident $k2:expr ) => ({ $me.reconsume = true; go!($me: to $s $k1 $k2); }); + + ( $me:ident : consume_char_ref ) => ({ $me.consume_char_ref(None); return ProcessResult::Continue; }); + ( $me:ident : consume_char_ref $addnl:expr ) => ({ $me.consume_char_ref(Some($addnl)); return ProcessResult::Continue; }); + + // We have a default next state after emitting a tag, but the sink can override. + ( $me:ident : emit_tag $s:ident ) => ({ + $me.state = states::$s; + return $me.emit_current_tag(); + }); + + ( $me:ident : eof ) => ({ $me.emit_eof(); return ProcessResult::Suspend; }); + + // If nothing else matched, it's a single command + ( $me:ident : $($cmd:tt)+ ) => ( sh_trace!($me: $($cmd)+); ); + + // or nothing. + ( $me:ident : ) => (()); +); + +macro_rules! go_match ( ( $me:ident : $x:expr, $($pats:pat),+ => $($cmds:tt)* ) => ( + match $x { + $($pats)|+ => go!($me: $($cmds)*), + _ => (), + } +)); + +// This is a macro because it can cause early return +// from the function where it is used. +macro_rules! get_char ( ($me:expr, $input:expr) => ( + unwrap_or_return!($me.get_char($input), ProcessResult::Suspend) +)); + +macro_rules! peek ( ($me:expr, $input:expr) => ( + unwrap_or_return!($me.peek($input), ProcessResult::Suspend) +)); + +macro_rules! pop_except_from ( ($me:expr, $input:expr, $set:expr) => ( + unwrap_or_return!($me.pop_except_from($input, $set), ProcessResult::Suspend) +)); + +macro_rules! eat ( ($me:expr, $input:expr, $pat:expr) => ( + unwrap_or_return!($me.eat($input, $pat, u8::eq_ignore_ascii_case), ProcessResult::Suspend) +)); + +macro_rules! eat_exact ( ($me:expr, $input:expr, $pat:expr) => ( + unwrap_or_return!($me.eat($input, $pat, u8::eq), ProcessResult::Suspend) +)); + +impl<Sink: TokenSink> Tokenizer<Sink> { + // Run the state machine for a while. + // Return true if we should be immediately re-invoked + // (this just simplifies control flow vs. break / continue). + #[allow(clippy::never_loop)] + fn step(&mut self, input: &mut BufferQueue) -> ProcessResult<Sink::Handle> { + if self.char_ref_tokenizer.is_some() { + return self.step_char_ref_tokenizer(input); + } + + debug!("processing in state {:?}", self.state); + match self.state { + //§ data-state + states::Data => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\0' '&' '<' '\n')) { + FromSet('\0') => go!(self: error; emit '\0'), + FromSet('&') => go!(self: consume_char_ref), + FromSet('<') => go!(self: to TagOpen), + FromSet(c) => go!(self: emit c), + NotFromSet(b) => self.emit_chars(b), + } + }, + + //§ rcdata-state + states::RawData(Rcdata) => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\0' '&' '<' '\n')) { + FromSet('\0') => go!(self: error; emit '\u{fffd}'), + FromSet('&') => go!(self: consume_char_ref), + FromSet('<') => go!(self: to RawLessThanSign Rcdata), + FromSet(c) => go!(self: emit c), + NotFromSet(b) => self.emit_chars(b), + } + }, + + //§ rawtext-state + states::RawData(Rawtext) => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\0' '<' '\n')) { + FromSet('\0') => go!(self: error; emit '\u{fffd}'), + FromSet('<') => go!(self: to RawLessThanSign Rawtext), + FromSet(c) => go!(self: emit c), + NotFromSet(b) => self.emit_chars(b), + } + }, + + //§ script-data-state + states::RawData(ScriptData) => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\0' '<' '\n')) { + FromSet('\0') => go!(self: error; emit '\u{fffd}'), + FromSet('<') => go!(self: to RawLessThanSign ScriptData), + FromSet(c) => go!(self: emit c), + NotFromSet(b) => self.emit_chars(b), + } + }, + + //§ script-data-escaped-state + states::RawData(ScriptDataEscaped(Escaped)) => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\0' '-' '<' '\n')) { + FromSet('\0') => go!(self: error; emit '\u{fffd}'), + FromSet('-') => go!(self: emit '-'; to ScriptDataEscapedDash Escaped), + FromSet('<') => go!(self: to RawLessThanSign ScriptDataEscaped Escaped), + FromSet(c) => go!(self: emit c), + NotFromSet(b) => self.emit_chars(b), + } + }, + + //§ script-data-double-escaped-state + states::RawData(ScriptDataEscaped(DoubleEscaped)) => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\0' '-' '<' '\n')) { + FromSet('\0') => go!(self: error; emit '\u{fffd}'), + FromSet('-') => go!(self: emit '-'; to ScriptDataEscapedDash DoubleEscaped), + FromSet('<') => { + go!(self: emit '<'; to RawLessThanSign ScriptDataEscaped DoubleEscaped) + }, + FromSet(c) => go!(self: emit c), + NotFromSet(b) => self.emit_chars(b), + } + }, + + //§ plaintext-state + states::Plaintext => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\0' '\n')) { + FromSet('\0') => go!(self: error; emit '\u{fffd}'), + FromSet(c) => go!(self: emit c), + NotFromSet(b) => self.emit_chars(b), + } + }, + + //§ tag-open-state + states::TagOpen => loop { + match get_char!(self, input) { + '!' => go!(self: clear_temp; to MarkupDeclarationOpen), + '/' => go!(self: to EndTagOpen), + '?' => go!(self: error; clear_comment; push_comment '?'; to BogusComment), + c => match lower_ascii_letter(c) { + Some(cl) => go!(self: create_tag StartTag cl; to TagName), + None => go!(self: error; emit '<'; reconsume Data), + }, + } + }, + + //§ end-tag-open-state + states::EndTagOpen => loop { + match get_char!(self, input) { + '>' => go!(self: error; to Data), + '\0' => { + go!(self: error; clear_comment; push_comment '\u{fffd}'; to BogusComment) + }, + c => match lower_ascii_letter(c) { + Some(cl) => go!(self: create_tag EndTag cl; to TagName), + None => go!(self: error; clear_comment; push_comment c; to BogusComment), + }, + } + }, + + //§ tag-name-state + states::TagName => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeAttributeName), + '/' => go!(self: to SelfClosingStartTag), + '>' => go!(self: emit_tag Data), + '\0' => go!(self: error; push_tag '\u{fffd}'), + c => go!(self: push_tag (c.to_ascii_lowercase())), + } + }, + + //§ script-data-escaped-less-than-sign-state + states::RawLessThanSign(ScriptDataEscaped(Escaped)) => loop { + match get_char!(self, input) { + '/' => go!(self: clear_temp; to RawEndTagOpen ScriptDataEscaped Escaped), + c => match lower_ascii_letter(c) { + Some(cl) => go!(self: clear_temp; push_temp cl; emit '<'; emit c; + to ScriptDataEscapeStart DoubleEscaped), + None => go!(self: emit '<'; reconsume RawData ScriptDataEscaped Escaped), + }, + } + }, + + //§ script-data-double-escaped-less-than-sign-state + states::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)) => loop { + match get_char!(self, input) { + '/' => go!(self: clear_temp; emit '/'; to ScriptDataDoubleEscapeEnd), + _ => go!(self: reconsume RawData ScriptDataEscaped DoubleEscaped), + } + }, + + //§ rcdata-less-than-sign-state rawtext-less-than-sign-state script-data-less-than-sign-state + // otherwise + states::RawLessThanSign(kind) => loop { + match get_char!(self, input) { + '/' => go!(self: clear_temp; to RawEndTagOpen kind), + '!' if kind == ScriptData => { + go!(self: emit '<'; emit '!'; to ScriptDataEscapeStart Escaped) + }, + _ => go!(self: emit '<'; reconsume RawData kind), + } + }, + + //§ rcdata-end-tag-open-state rawtext-end-tag-open-state script-data-end-tag-open-state script-data-escaped-end-tag-open-state + states::RawEndTagOpen(kind) => loop { + let c = get_char!(self, input); + match lower_ascii_letter(c) { + Some(cl) => go!(self: create_tag EndTag cl; push_temp c; to RawEndTagName kind), + None => go!(self: emit '<'; emit '/'; reconsume RawData kind), + } + }, + + //§ rcdata-end-tag-name-state rawtext-end-tag-name-state script-data-end-tag-name-state script-data-escaped-end-tag-name-state + states::RawEndTagName(kind) => loop { + let c = get_char!(self, input); + if self.have_appropriate_end_tag() { + match c { + '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeAttributeName), + '/' => go!(self: to SelfClosingStartTag), + '>' => go!(self: emit_tag Data), + _ => (), + } + } + + match lower_ascii_letter(c) { + Some(cl) => go!(self: push_tag cl; push_temp c), + None => { + go!(self: discard_tag; emit '<'; emit '/'; emit_temp; reconsume RawData kind) + }, + } + }, + + //§ script-data-double-escape-start-state + states::ScriptDataEscapeStart(DoubleEscaped) => loop { + let c = get_char!(self, input); + match c { + '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => { + let esc = if &*self.temp_buf == "script" { + DoubleEscaped + } else { + Escaped + }; + go!(self: emit c; to RawData ScriptDataEscaped esc); + }, + _ => match lower_ascii_letter(c) { + Some(cl) => go!(self: push_temp cl; emit c), + None => go!(self: reconsume RawData ScriptDataEscaped Escaped), + }, + } + }, + + //§ script-data-escape-start-state + states::ScriptDataEscapeStart(Escaped) => loop { + match get_char!(self, input) { + '-' => go!(self: emit '-'; to ScriptDataEscapeStartDash), + _ => go!(self: reconsume RawData ScriptData), + } + }, + + //§ script-data-escape-start-dash-state + states::ScriptDataEscapeStartDash => loop { + match get_char!(self, input) { + '-' => go!(self: emit '-'; to ScriptDataEscapedDashDash Escaped), + _ => go!(self: reconsume RawData ScriptData), + } + }, + + //§ script-data-escaped-dash-state script-data-double-escaped-dash-state + states::ScriptDataEscapedDash(kind) => loop { + match get_char!(self, input) { + '-' => go!(self: emit '-'; to ScriptDataEscapedDashDash kind), + '<' => { + if kind == DoubleEscaped { + go!(self: emit '<'); + } + go!(self: to RawLessThanSign ScriptDataEscaped kind); + }, + '\0' => go!(self: error; emit '\u{fffd}'; to RawData ScriptDataEscaped kind), + c => go!(self: emit c; to RawData ScriptDataEscaped kind), + } + }, + + //§ script-data-escaped-dash-dash-state script-data-double-escaped-dash-dash-state + states::ScriptDataEscapedDashDash(kind) => loop { + match get_char!(self, input) { + '-' => go!(self: emit '-'), + '<' => { + if kind == DoubleEscaped { + go!(self: emit '<'); + } + go!(self: to RawLessThanSign ScriptDataEscaped kind); + }, + '>' => go!(self: emit '>'; to RawData ScriptData), + '\0' => go!(self: error; emit '\u{fffd}'; to RawData ScriptDataEscaped kind), + c => go!(self: emit c; to RawData ScriptDataEscaped kind), + } + }, + + //§ script-data-double-escape-end-state + states::ScriptDataDoubleEscapeEnd => loop { + let c = get_char!(self, input); + match c { + '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => { + let esc = if &*self.temp_buf == "script" { + Escaped + } else { + DoubleEscaped + }; + go!(self: emit c; to RawData ScriptDataEscaped esc); + }, + _ => match lower_ascii_letter(c) { + Some(cl) => go!(self: push_temp cl; emit c), + None => go!(self: reconsume RawData ScriptDataEscaped DoubleEscaped), + }, + } + }, + + //§ before-attribute-name-state + states::BeforeAttributeName => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => (), + '/' => go!(self: to SelfClosingStartTag), + '>' => go!(self: emit_tag Data), + '\0' => go!(self: error; create_attr '\u{fffd}'; to AttributeName), + c => match lower_ascii_letter(c) { + Some(cl) => go!(self: create_attr cl; to AttributeName), + None => { + go_match!(self: c, + '"' , '\'' , '<' , '=' => error); + go!(self: create_attr c; to AttributeName); + }, + }, + } + }, + + //§ attribute-name-state + states::AttributeName => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => go!(self: to AfterAttributeName), + '/' => go!(self: to SelfClosingStartTag), + '=' => go!(self: to BeforeAttributeValue), + '>' => go!(self: emit_tag Data), + '\0' => go!(self: error; push_name '\u{fffd}'), + c => match lower_ascii_letter(c) { + Some(cl) => go!(self: push_name cl), + None => { + go_match!(self: c, + '"' , '\'' , '<' => error); + go!(self: push_name c); + }, + }, + } + }, + + //§ after-attribute-name-state + states::AfterAttributeName => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => (), + '/' => go!(self: to SelfClosingStartTag), + '=' => go!(self: to BeforeAttributeValue), + '>' => go!(self: emit_tag Data), + '\0' => go!(self: error; create_attr '\u{fffd}'; to AttributeName), + c => match lower_ascii_letter(c) { + Some(cl) => go!(self: create_attr cl; to AttributeName), + None => { + go_match!(self: c, + '"' , '\'' , '<' => error); + go!(self: create_attr c; to AttributeName); + }, + }, + } + }, + + //§ before-attribute-value-state + // Use peek so we can handle the first attr character along with the rest, + // hopefully in the same zero-copy buffer. + states::BeforeAttributeValue => loop { + match peek!(self, input) { + '\t' | '\n' | '\r' | '\x0C' | ' ' => go!(self: discard_char input), + '"' => go!(self: discard_char input; to AttributeValue DoubleQuoted), + '\'' => go!(self: discard_char input; to AttributeValue SingleQuoted), + '\0' => { + go!(self: discard_char input; error; push_value '\u{fffd}'; to AttributeValue Unquoted) + }, + '>' => go!(self: discard_char input; error; emit_tag Data), + _ => go!(self: to AttributeValue Unquoted), + } + }, + + //§ attribute-value-(double-quoted)-state + states::AttributeValue(DoubleQuoted) => loop { + match pop_except_from!(self, input, small_char_set!('\r' '"' '&' '\0' '\n')) { + FromSet('"') => go!(self: to AfterAttributeValueQuoted), + FromSet('&') => go!(self: consume_char_ref '"'), + FromSet('\0') => go!(self: error; push_value '\u{fffd}'), + FromSet(c) => go!(self: push_value c), + NotFromSet(ref b) => go!(self: append_value b), + } + }, + + //§ attribute-value-(single-quoted)-state + states::AttributeValue(SingleQuoted) => loop { + match pop_except_from!(self, input, small_char_set!('\r' '\'' '&' '\0' '\n')) { + FromSet('\'') => go!(self: to AfterAttributeValueQuoted), + FromSet('&') => go!(self: consume_char_ref '\''), + FromSet('\0') => go!(self: error; push_value '\u{fffd}'), + FromSet(c) => go!(self: push_value c), + NotFromSet(ref b) => go!(self: append_value b), + } + }, + + //§ attribute-value-(unquoted)-state + states::AttributeValue(Unquoted) => loop { + match pop_except_from!( + self, + input, + small_char_set!('\r' '\t' '\n' '\x0C' ' ' '&' '>' '\0') + ) { + FromSet('\t') | FromSet('\n') | FromSet('\x0C') | FromSet(' ') => { + go!(self: to BeforeAttributeName) + }, + FromSet('&') => go!(self: consume_char_ref '>'), + FromSet('>') => go!(self: emit_tag Data), + FromSet('\0') => go!(self: error; push_value '\u{fffd}'), + FromSet(c) => { + go_match!(self: c, + '"' , '\'' , '<' , '=' , '`' => error); + go!(self: push_value c); + }, + NotFromSet(ref b) => go!(self: append_value b), + } + }, + + //§ after-attribute-value-(quoted)-state + states::AfterAttributeValueQuoted => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeAttributeName), + '/' => go!(self: to SelfClosingStartTag), + '>' => go!(self: emit_tag Data), + _ => go!(self: error; reconsume BeforeAttributeName), + } + }, + + //§ self-closing-start-tag-state + states::SelfClosingStartTag => loop { + match get_char!(self, input) { + '>' => { + self.current_tag_self_closing = true; + go!(self: emit_tag Data); + }, + _ => go!(self: error; reconsume BeforeAttributeName), + } + }, + + //§ comment-start-state + states::CommentStart => loop { + match get_char!(self, input) { + '-' => go!(self: to CommentStartDash), + '\0' => go!(self: error; push_comment '\u{fffd}'; to Comment), + '>' => go!(self: error; emit_comment; to Data), + c => go!(self: push_comment c; to Comment), + } + }, + + //§ comment-start-dash-state + states::CommentStartDash => loop { + match get_char!(self, input) { + '-' => go!(self: to CommentEnd), + '\0' => go!(self: error; append_comment "-\u{fffd}"; to Comment), + '>' => go!(self: error; emit_comment; to Data), + c => go!(self: push_comment '-'; push_comment c; to Comment), + } + }, + + //§ comment-state + states::Comment => loop { + match get_char!(self, input) { + '-' => go!(self: to CommentEndDash), + '\0' => go!(self: error; push_comment '\u{fffd}'), + c => go!(self: push_comment c), + } + }, + + //§ comment-end-dash-state + states::CommentEndDash => loop { + match get_char!(self, input) { + '-' => go!(self: to CommentEnd), + '\0' => go!(self: error; append_comment "-\u{fffd}"; to Comment), + c => go!(self: push_comment '-'; push_comment c; to Comment), + } + }, + + //§ comment-end-state + states::CommentEnd => loop { + match get_char!(self, input) { + '>' => go!(self: emit_comment; to Data), + '\0' => go!(self: error; append_comment "--\u{fffd}"; to Comment), + '!' => go!(self: error; to CommentEndBang), + '-' => go!(self: error; push_comment '-'), + c => go!(self: error; append_comment "--"; push_comment c; to Comment), + } + }, + + //§ comment-end-bang-state + states::CommentEndBang => loop { + match get_char!(self, input) { + '-' => go!(self: append_comment "--!"; to CommentEndDash), + '>' => go!(self: emit_comment; to Data), + '\0' => go!(self: error; append_comment "--!\u{fffd}"; to Comment), + c => go!(self: append_comment "--!"; push_comment c; to Comment), + } + }, + + //§ doctype-state + states::Doctype => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeName), + _ => go!(self: error; reconsume BeforeDoctypeName), + } + }, + + //§ before-doctype-name-state + states::BeforeDoctypeName => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => (), + '\0' => { + go!(self: error; create_doctype; push_doctype_name '\u{fffd}'; to DoctypeName) + }, + '>' => go!(self: error; create_doctype; force_quirks; emit_doctype; to Data), + c => go!(self: create_doctype; push_doctype_name (c.to_ascii_lowercase()); + to DoctypeName), + } + }, + + //§ doctype-name-state + states::DoctypeName => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => go!(self: clear_temp; to AfterDoctypeName), + '>' => go!(self: emit_doctype; to Data), + '\0' => go!(self: error; push_doctype_name '\u{fffd}'), + c => go!(self: push_doctype_name (c.to_ascii_lowercase())), + } + }, + + //§ after-doctype-name-state + states::AfterDoctypeName => loop { + if eat!(self, input, "public") { + go!(self: to AfterDoctypeKeyword Public); + } else if eat!(self, input, "system") { + go!(self: to AfterDoctypeKeyword System); + } else { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => (), + '>' => go!(self: emit_doctype; to Data), + _ => go!(self: error; force_quirks; to BogusDoctype), + } + } + }, + + //§ after-doctype-public-keyword-state after-doctype-system-keyword-state + states::AfterDoctypeKeyword(kind) => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => go!(self: to BeforeDoctypeIdentifier kind), + '"' => { + go!(self: error; clear_doctype_id kind; to DoctypeIdentifierDoubleQuoted kind) + }, + '\'' => { + go!(self: error; clear_doctype_id kind; to DoctypeIdentifierSingleQuoted kind) + }, + '>' => go!(self: error; force_quirks; emit_doctype; to Data), + _ => go!(self: error; force_quirks; to BogusDoctype), + } + }, + + //§ before-doctype-public-identifier-state before-doctype-system-identifier-state + states::BeforeDoctypeIdentifier(kind) => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => (), + '"' => go!(self: clear_doctype_id kind; to DoctypeIdentifierDoubleQuoted kind), + '\'' => go!(self: clear_doctype_id kind; to DoctypeIdentifierSingleQuoted kind), + '>' => go!(self: error; force_quirks; emit_doctype; to Data), + _ => go!(self: error; force_quirks; to BogusDoctype), + } + }, + + //§ doctype-public-identifier-(double-quoted)-state doctype-system-identifier-(double-quoted)-state + states::DoctypeIdentifierDoubleQuoted(kind) => loop { + match get_char!(self, input) { + '"' => go!(self: to AfterDoctypeIdentifier kind), + '\0' => go!(self: error; push_doctype_id kind '\u{fffd}'), + '>' => go!(self: error; force_quirks; emit_doctype; to Data), + c => go!(self: push_doctype_id kind c), + } + }, + + //§ doctype-public-identifier-(single-quoted)-state doctype-system-identifier-(single-quoted)-state + states::DoctypeIdentifierSingleQuoted(kind) => loop { + match get_char!(self, input) { + '\'' => go!(self: to AfterDoctypeIdentifier kind), + '\0' => go!(self: error; push_doctype_id kind '\u{fffd}'), + '>' => go!(self: error; force_quirks; emit_doctype; to Data), + c => go!(self: push_doctype_id kind c), + } + }, + + //§ after-doctype-public-identifier-state + states::AfterDoctypeIdentifier(Public) => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => { + go!(self: to BetweenDoctypePublicAndSystemIdentifiers) + }, + '>' => go!(self: emit_doctype; to Data), + '"' => { + go!(self: error; clear_doctype_id System; to DoctypeIdentifierDoubleQuoted System) + }, + '\'' => { + go!(self: error; clear_doctype_id System; to DoctypeIdentifierSingleQuoted System) + }, + _ => go!(self: error; force_quirks; to BogusDoctype), + } + }, + + //§ after-doctype-system-identifier-state + states::AfterDoctypeIdentifier(System) => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => (), + '>' => go!(self: emit_doctype; to Data), + _ => go!(self: error; to BogusDoctype), + } + }, + + //§ between-doctype-public-and-system-identifiers-state + states::BetweenDoctypePublicAndSystemIdentifiers => loop { + match get_char!(self, input) { + '\t' | '\n' | '\x0C' | ' ' => (), + '>' => go!(self: emit_doctype; to Data), + '"' => { + go!(self: clear_doctype_id System; to DoctypeIdentifierDoubleQuoted System) + }, + '\'' => { + go!(self: clear_doctype_id System; to DoctypeIdentifierSingleQuoted System) + }, + _ => go!(self: error; force_quirks; to BogusDoctype), + } + }, + + //§ bogus-doctype-state + states::BogusDoctype => loop { + match get_char!(self, input) { + '>' => go!(self: emit_doctype; to Data), + _ => (), + } + }, + + //§ bogus-comment-state + states::BogusComment => loop { + match get_char!(self, input) { + '>' => go!(self: emit_comment; to Data), + '\0' => go!(self: push_comment '\u{fffd}'), + c => go!(self: push_comment c), + } + }, + + //§ markup-declaration-open-state + states::MarkupDeclarationOpen => loop { + if eat_exact!(self, input, "--") { + go!(self: clear_comment; to CommentStart); + } else if eat!(self, input, "doctype") { + go!(self: to Doctype); + } else { + if self + .sink + .adjusted_current_node_present_but_not_in_html_namespace() + { + if eat_exact!(self, input, "[CDATA[") { + go!(self: clear_temp; to CdataSection); + } + } + go!(self: error; to BogusComment); + } + }, + + //§ cdata-section-state + states::CdataSection => loop { + match get_char!(self, input) { + ']' => go!(self: to CdataSectionBracket), + '\0' => go!(self: emit_temp; emit '\0'), + c => go!(self: push_temp c), + } + }, + + //§ cdata-section-bracket + states::CdataSectionBracket => match get_char!(self, input) { + ']' => go!(self: to CdataSectionEnd), + _ => go!(self: push_temp ']'; reconsume CdataSection), + }, + + //§ cdata-section-end + states::CdataSectionEnd => loop { + match get_char!(self, input) { + ']' => go!(self: push_temp ']'), + '>' => go!(self: emit_temp; to Data), + _ => go!(self: push_temp ']'; push_temp ']'; reconsume CdataSection), + } + }, + //§ END + } + } + + fn step_char_ref_tokenizer(&mut self, input: &mut BufferQueue) -> ProcessResult<Sink::Handle> { + // FIXME HACK: Take and replace the tokenizer so we don't + // double-mut-borrow self. This is why it's boxed. + let mut tok = self.char_ref_tokenizer.take().unwrap(); + let outcome = tok.step(self, input); + + let progress = match outcome { + char_ref::Done => { + self.process_char_ref(tok.get_result()); + return ProcessResult::Continue; + }, + + char_ref::Stuck => ProcessResult::Suspend, + char_ref::Progress => ProcessResult::Continue, + }; + + self.char_ref_tokenizer = Some(tok); + progress + } + + fn process_char_ref(&mut self, char_ref: CharRef) { + let CharRef { + mut chars, + mut num_chars, + } = char_ref; + + if num_chars == 0 { + chars[0] = '&'; + num_chars = 1; + } + + for i in 0..num_chars { + let c = chars[i as usize]; + match self.state { + states::Data | states::RawData(states::Rcdata) => go!(self: emit c), + + states::AttributeValue(_) => go!(self: push_value c), + + _ => panic!( + "state {:?} should not be reachable in process_char_ref", + self.state + ), + } + } + } + + /// Indicate that we have reached the end of the input. + pub fn end(&mut self) { + // Handle EOF in the char ref sub-tokenizer, if there is one. + // Do this first because it might un-consume stuff. + let mut input = BufferQueue::new(); + match self.char_ref_tokenizer.take() { + None => (), + Some(mut tok) => { + tok.end_of_file(self, &mut input); + self.process_char_ref(tok.get_result()); + }, + } + + // Process all remaining buffered input. + // If we're waiting for lookahead, we're not gonna get it. + self.at_eof = true; + assert!(matches!(self.run(&mut input), TokenizerResult::Done)); + assert!(input.is_empty()); + + loop { + match self.eof_step() { + ProcessResult::Continue => (), + ProcessResult::Suspend => break, + ProcessResult::Script(_) => unreachable!(), + } + } + + self.sink.end(); + + if self.opts.profile { + self.dump_profile(); + } + } + + fn dump_profile(&self) { + let mut results: Vec<(states::State, u64)> = + self.state_profile.iter().map(|(s, t)| (*s, *t)).collect(); + results.sort_by(|&(_, x), &(_, y)| y.cmp(&x)); + + let total: u64 = results + .iter() + .map(|&(_, t)| t) + .fold(0, ::std::ops::Add::add); + println!("\nTokenizer profile, in nanoseconds"); + println!("\n{:12} total in token sink", self.time_in_sink); + println!("\n{:12} total in tokenizer", total); + + for (k, v) in results.into_iter() { + let pct = 100.0 * (v as f64) / (total as f64); + println!("{:12} {:4.1}% {:?}", v, pct, k); + } + } + + fn eof_step(&mut self) -> ProcessResult<Sink::Handle> { + debug!("processing EOF in state {:?}", self.state); + match self.state { + states::Data | + states::RawData(Rcdata) | + states::RawData(Rawtext) | + states::RawData(ScriptData) | + states::Plaintext => go!(self: eof), + + states::TagName | + states::RawData(ScriptDataEscaped(_)) | + states::BeforeAttributeName | + states::AttributeName | + states::AfterAttributeName | + states::BeforeAttributeValue | + states::AttributeValue(_) | + states::AfterAttributeValueQuoted | + states::SelfClosingStartTag | + states::ScriptDataEscapedDash(_) | + states::ScriptDataEscapedDashDash(_) => go!(self: error_eof; to Data), + + states::TagOpen => go!(self: error_eof; emit '<'; to Data), + + states::EndTagOpen => go!(self: error_eof; emit '<'; emit '/'; to Data), + + states::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)) => { + go!(self: to RawData ScriptDataEscaped DoubleEscaped) + }, + + states::RawLessThanSign(kind) => go!(self: emit '<'; to RawData kind), + + states::RawEndTagOpen(kind) => go!(self: emit '<'; emit '/'; to RawData kind), + + states::RawEndTagName(kind) => { + go!(self: emit '<'; emit '/'; emit_temp; to RawData kind) + }, + + states::ScriptDataEscapeStart(kind) => go!(self: to RawData ScriptDataEscaped kind), + + states::ScriptDataEscapeStartDash => go!(self: to RawData ScriptData), + + states::ScriptDataDoubleEscapeEnd => { + go!(self: to RawData ScriptDataEscaped DoubleEscaped) + }, + + states::CommentStart | + states::CommentStartDash | + states::Comment | + states::CommentEndDash | + states::CommentEnd | + states::CommentEndBang => go!(self: error_eof; emit_comment; to Data), + + states::Doctype | states::BeforeDoctypeName => { + go!(self: error_eof; create_doctype; force_quirks; emit_doctype; to Data) + }, + + states::DoctypeName | + states::AfterDoctypeName | + states::AfterDoctypeKeyword(_) | + states::BeforeDoctypeIdentifier(_) | + states::DoctypeIdentifierDoubleQuoted(_) | + states::DoctypeIdentifierSingleQuoted(_) | + states::AfterDoctypeIdentifier(_) | + states::BetweenDoctypePublicAndSystemIdentifiers => { + go!(self: error_eof; force_quirks; emit_doctype; to Data) + }, + + states::BogusDoctype => go!(self: emit_doctype; to Data), + + states::BogusComment => go!(self: emit_comment; to Data), + + states::MarkupDeclarationOpen => go!(self: error; to BogusComment), + + states::CdataSection => go!(self: emit_temp; error_eof; to Data), + + states::CdataSectionBracket => go!(self: push_temp ']'; to CdataSection), + + states::CdataSectionEnd => go!(self: push_temp ']'; push_temp ']'; to CdataSection), + } + } +} + +#[cfg(test)] +#[allow(non_snake_case)] +mod test { + use super::option_push; // private items + use crate::tendril::{SliceExt, StrTendril}; + + use super::{TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts}; + + use super::interface::{CharacterTokens, EOFToken, NullCharacterToken, ParseError}; + use super::interface::{EndTag, StartTag, Tag, TagKind}; + use super::interface::{TagToken, Token}; + + use markup5ever::buffer_queue::BufferQueue; + use std::mem::replace; + + use crate::LocalName; + + // LinesMatch implements the TokenSink trait. It is used for testing to see + // if current_line is being updated when process_token is called. The lines + // vector is a collection of the line numbers that each token is on. + struct LinesMatch { + tokens: Vec<Token>, + current_str: StrTendril, + lines: Vec<(Token, u64)>, + } + + impl LinesMatch { + fn new() -> LinesMatch { + LinesMatch { + tokens: vec![], + current_str: StrTendril::new(), + lines: vec![], + } + } + + fn push(&mut self, token: Token, line_number: u64) { + self.finish_str(); + self.lines.push((token, line_number)); + } + + fn finish_str(&mut self) { + if self.current_str.len() > 0 { + let s = replace(&mut self.current_str, StrTendril::new()); + self.tokens.push(CharacterTokens(s)); + } + } + } + + impl TokenSink for LinesMatch { + type Handle = (); + + fn process_token( + &mut self, + token: Token, + line_number: u64, + ) -> TokenSinkResult<Self::Handle> { + match token { + CharacterTokens(b) => { + self.current_str.push_slice(&b); + }, + + NullCharacterToken => { + self.current_str.push_char('\0'); + }, + + ParseError(_) => { + panic!("unexpected parse error"); + }, + + TagToken(mut t) => { + // The spec seems to indicate that one can emit + // erroneous end tags with attrs, but the test + // cases don't contain them. + match t.kind { + EndTag => { + t.self_closing = false; + t.attrs = vec![]; + }, + _ => t.attrs.sort_by(|a1, a2| a1.name.cmp(&a2.name)), + } + self.push(TagToken(t), line_number); + }, + + EOFToken => (), + + _ => self.push(token, line_number), + } + TokenSinkResult::Continue + } + } + + // Take in tokens, process them, and return vector with line + // numbers that each token is on + fn tokenize(input: Vec<StrTendril>, opts: TokenizerOpts) -> Vec<(Token, u64)> { + let sink = LinesMatch::new(); + let mut tok = Tokenizer::new(sink, opts); + let mut buffer = BufferQueue::new(); + for chunk in input.into_iter() { + buffer.push_back(chunk); + let _ = tok.feed(&mut buffer); + } + tok.end(); + tok.sink.lines + } + + // Create a tag token + fn create_tag(token: StrTendril, tagkind: TagKind) -> Token { + let name = LocalName::from(&*token); + let token = TagToken(Tag { + kind: tagkind, + name, + self_closing: false, + attrs: vec![], + }); + token + } + + #[test] + fn push_to_None_gives_singleton() { + let mut s: Option<StrTendril> = None; + option_push(&mut s, 'x'); + assert_eq!(s, Some("x".to_tendril())); + } + + #[test] + fn push_to_empty_appends() { + let mut s: Option<StrTendril> = Some(StrTendril::new()); + option_push(&mut s, 'x'); + assert_eq!(s, Some("x".to_tendril())); + } + + #[test] + fn push_to_nonempty_appends() { + let mut s: Option<StrTendril> = Some(StrTendril::from_slice("y")); + option_push(&mut s, 'x'); + assert_eq!(s, Some("yx".to_tendril())); + } + + #[test] + fn check_lines() { + let opts = TokenizerOpts { + exact_errors: false, + discard_bom: true, + profile: false, + initial_state: None, + last_start_tag_name: None, + }; + let vector = vec![ + StrTendril::from("<a>\n"), + StrTendril::from("<b>\n"), + StrTendril::from("</b>\n"), + StrTendril::from("</a>\n"), + ]; + let expected = vec![ + (create_tag(StrTendril::from("a"), StartTag), 1), + (create_tag(StrTendril::from("b"), StartTag), 2), + (create_tag(StrTendril::from("b"), EndTag), 3), + (create_tag(StrTendril::from("a"), EndTag), 4), + ]; + let results = tokenize(vector, opts); + assert_eq!(results, expected); + } + + #[test] + fn check_lines_with_new_line() { + let opts = TokenizerOpts { + exact_errors: false, + discard_bom: true, + profile: false, + initial_state: None, + last_start_tag_name: None, + }; + let vector = vec![ + StrTendril::from("<a>\r\n"), + StrTendril::from("<b>\r\n"), + StrTendril::from("</b>\r\n"), + StrTendril::from("</a>\r\n"), + ]; + let expected = vec![ + (create_tag(StrTendril::from("a"), StartTag), 1), + (create_tag(StrTendril::from("b"), StartTag), 2), + (create_tag(StrTendril::from("b"), EndTag), 3), + (create_tag(StrTendril::from("a"), EndTag), 4), + ]; + let results = tokenize(vector, opts); + assert_eq!(results, expected); + } +} diff --git a/src/tokenizer/states.rs b/src/tokenizer/states.rs new file mode 100644 index 0000000..d455e9a --- /dev/null +++ b/src/tokenizer/states.rs @@ -0,0 +1,93 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Tokenizer states. +//! +//! This is public for use by the tokenizer tests. Other library +//! users should not have to care about this. + +pub use self::AttrValueKind::*; +pub use self::DoctypeIdKind::*; +pub use self::RawKind::*; +pub use self::ScriptEscapeKind::*; +pub use self::State::*; + +#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] +pub enum ScriptEscapeKind { + Escaped, + DoubleEscaped, +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] +pub enum DoctypeIdKind { + Public, + System, +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] +pub enum RawKind { + Rcdata, + Rawtext, + ScriptData, + ScriptDataEscaped(ScriptEscapeKind), +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] +pub enum AttrValueKind { + Unquoted, + SingleQuoted, + DoubleQuoted, +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Copy, Clone, Hash, Debug)] +pub enum State { + Data, + Plaintext, + TagOpen, + EndTagOpen, + TagName, + RawData(RawKind), + RawLessThanSign(RawKind), + RawEndTagOpen(RawKind), + RawEndTagName(RawKind), + ScriptDataEscapeStart(ScriptEscapeKind), + ScriptDataEscapeStartDash, + ScriptDataEscapedDash(ScriptEscapeKind), + ScriptDataEscapedDashDash(ScriptEscapeKind), + ScriptDataDoubleEscapeEnd, + BeforeAttributeName, + AttributeName, + AfterAttributeName, + BeforeAttributeValue, + AttributeValue(AttrValueKind), + AfterAttributeValueQuoted, + SelfClosingStartTag, + BogusComment, + MarkupDeclarationOpen, + CommentStart, + CommentStartDash, + Comment, + CommentEndDash, + CommentEnd, + CommentEndBang, + Doctype, + BeforeDoctypeName, + DoctypeName, + AfterDoctypeName, + AfterDoctypeKeyword(DoctypeIdKind), + BeforeDoctypeIdentifier(DoctypeIdKind), + DoctypeIdentifierDoubleQuoted(DoctypeIdKind), + DoctypeIdentifierSingleQuoted(DoctypeIdKind), + AfterDoctypeIdentifier(DoctypeIdKind), + BetweenDoctypePublicAndSystemIdentifiers, + BogusDoctype, + CdataSection, + CdataSectionBracket, + CdataSectionEnd, +} diff --git a/src/tree_builder/data.rs b/src/tree_builder/data.rs new file mode 100644 index 0000000..9d51a71 --- /dev/null +++ b/src/tree_builder/data.rs @@ -0,0 +1,171 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use crate::interface::{LimitedQuirks, NoQuirks, Quirks, QuirksMode}; +use crate::tendril::StrTendril; +use crate::tokenizer::Doctype; + +// These should all be lowercase, for ASCII-case-insensitive matching. +static QUIRKY_PUBLIC_PREFIXES: &'static [&'static str] = &[ + "-//advasoft ltd//dtd html 3.0 aswedit + extensions//", + "-//as//dtd html 3.0 aswedit + extensions//", + "-//ietf//dtd html 2.0 level 1//", + "-//ietf//dtd html 2.0 level 2//", + "-//ietf//dtd html 2.0 strict level 1//", + "-//ietf//dtd html 2.0 strict level 2//", + "-//ietf//dtd html 2.0 strict//", + "-//ietf//dtd html 2.0//", + "-//ietf//dtd html 2.1e//", + "-//ietf//dtd html 3.0//", + "-//ietf//dtd html 3.2 final//", + "-//ietf//dtd html 3.2//", + "-//ietf//dtd html 3//", + "-//ietf//dtd html level 0//", + "-//ietf//dtd html level 1//", + "-//ietf//dtd html level 2//", + "-//ietf//dtd html level 3//", + "-//ietf//dtd html strict level 0//", + "-//ietf//dtd html strict level 1//", + "-//ietf//dtd html strict level 2//", + "-//ietf//dtd html strict level 3//", + "-//ietf//dtd html strict//", + "-//ietf//dtd html//", + "-//metrius//dtd metrius presentational//", + "-//microsoft//dtd internet explorer 2.0 html strict//", + "-//microsoft//dtd internet explorer 2.0 html//", + "-//microsoft//dtd internet explorer 2.0 tables//", + "-//microsoft//dtd internet explorer 3.0 html strict//", + "-//microsoft//dtd internet explorer 3.0 html//", + "-//microsoft//dtd internet explorer 3.0 tables//", + "-//netscape comm. corp.//dtd html//", + "-//netscape comm. corp.//dtd strict html//", + "-//o'reilly and associates//dtd html 2.0//", + "-//o'reilly and associates//dtd html extended 1.0//", + "-//o'reilly and associates//dtd html extended relaxed 1.0//", + "-//softquad software//dtd hotmetal pro 6.0::19990601::extensions to html 4.0//", + "-//softquad//dtd hotmetal pro 4.0::19971010::extensions to html 4.0//", + "-//spyglass//dtd html 2.0 extended//", + "-//sq//dtd html 2.0 hotmetal + extensions//", + "-//sun microsystems corp.//dtd hotjava html//", + "-//sun microsystems corp.//dtd hotjava strict html//", + "-//w3c//dtd html 3 1995-03-24//", + "-//w3c//dtd html 3.2 draft//", + "-//w3c//dtd html 3.2 final//", + "-//w3c//dtd html 3.2//", + "-//w3c//dtd html 3.2s draft//", + "-//w3c//dtd html 4.0 frameset//", + "-//w3c//dtd html 4.0 transitional//", + "-//w3c//dtd html experimental 19960712//", + "-//w3c//dtd html experimental 970421//", + "-//w3c//dtd w3 html//", + "-//w3o//dtd w3 html 3.0//", + "-//webtechs//dtd mozilla html 2.0//", + "-//webtechs//dtd mozilla html//", +]; + +static QUIRKY_PUBLIC_MATCHES: &'static [&'static str] = &[ + "-//w3o//dtd w3 html strict 3.0//en//", + "-/w3c/dtd html 4.0 transitional/en", + "html", +]; + +static QUIRKY_SYSTEM_MATCHES: &'static [&'static str] = + &["http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd"]; + +static LIMITED_QUIRKY_PUBLIC_PREFIXES: &'static [&'static str] = &[ + "-//w3c//dtd xhtml 1.0 frameset//", + "-//w3c//dtd xhtml 1.0 transitional//", +]; + +static HTML4_PUBLIC_PREFIXES: &'static [&'static str] = &[ + "-//w3c//dtd html 4.01 frameset//", + "-//w3c//dtd html 4.01 transitional//", +]; + +pub fn doctype_error_and_quirks(doctype: &Doctype, iframe_srcdoc: bool) -> (bool, QuirksMode) { + fn opt_string_as_slice<'t>(x: &'t Option<String>) -> Option<&'t str> { + x.as_ref().map(|y| &y[..]) + } + + fn opt_tendril_as_slice<'t>(x: &'t Option<StrTendril>) -> Option<&'t str> { + match *x { + Some(ref t) => Some(t), + None => None, + } + } + + fn opt_to_ascii_lower(x: Option<&str>) -> Option<String> { + x.map(|y| y.to_ascii_lowercase()) + } + + let name = opt_tendril_as_slice(&doctype.name); + let public = opt_tendril_as_slice(&doctype.public_id); + let system = opt_tendril_as_slice(&doctype.system_id); + + let err = match (name, public, system) { + (Some("html"), None, None) | + (Some("html"), None, Some("about:legacy-compat")) | + (Some("html"), Some("-//W3C//DTD HTML 4.0//EN"), None) | + ( + Some("html"), + Some("-//W3C//DTD HTML 4.0//EN"), + Some("http://www.w3.org/TR/REC-html40/strict.dtd"), + ) | + (Some("html"), Some("-//W3C//DTD HTML 4.01//EN"), None) | + ( + Some("html"), + Some("-//W3C//DTD HTML 4.01//EN"), + Some("http://www.w3.org/TR/html4/strict.dtd"), + ) | + ( + Some("html"), + Some("-//W3C//DTD XHTML 1.0 Strict//EN"), + Some("http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"), + ) | + ( + Some("html"), + Some("-//W3C//DTD XHTML 1.1//EN"), + Some("http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"), + ) => false, + + _ => true, + }; + + // FIXME: We could do something asymptotically faster here. + // But there aren't many strings, and this happens at most once per parse. + fn contains_pfx(haystack: &[&str], needle: &str) -> bool { + haystack.iter().any(|&x| needle.starts_with(x)) + } + + // Quirks-mode matches are case-insensitive. + let public = opt_to_ascii_lower(public); + let system = opt_to_ascii_lower(system); + + let quirk = match (opt_string_as_slice(&public), opt_string_as_slice(&system)) { + _ if doctype.force_quirks => Quirks, + _ if name != Some("html") => Quirks, + + _ if iframe_srcdoc => NoQuirks, + + (Some(ref p), _) if QUIRKY_PUBLIC_MATCHES.contains(p) => Quirks, + (_, Some(ref s)) if QUIRKY_SYSTEM_MATCHES.contains(s) => Quirks, + + (Some(p), _) if contains_pfx(QUIRKY_PUBLIC_PREFIXES, p) => Quirks, + (Some(p), _) if contains_pfx(LIMITED_QUIRKY_PUBLIC_PREFIXES, p) => LimitedQuirks, + + (Some(p), s) if contains_pfx(HTML4_PUBLIC_PREFIXES, p) => match s { + None => Quirks, + Some(_) => LimitedQuirks, + }, + + _ => NoQuirks, + }; + + (err, quirk) +} diff --git a/src/tree_builder/mod.rs b/src/tree_builder/mod.rs new file mode 100644 index 0000000..a6fa8bf --- /dev/null +++ b/src/tree_builder/mod.rs @@ -0,0 +1,1681 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#![allow(warnings)] + +//! The HTML5 tree builder. + +pub use crate::interface::{create_element, ElementFlags, NextParserState, Tracer, TreeSink}; +pub use crate::interface::{AppendNode, AppendText, Attribute, NodeOrText}; +pub use crate::interface::{LimitedQuirks, NoQuirks, Quirks, QuirksMode}; + +use self::types::*; + +use crate::tendril::StrTendril; +use crate::{ExpandedName, LocalName, Namespace, QualName}; + +use crate::tokenizer; +use crate::tokenizer::states as tok_state; +use crate::tokenizer::{Doctype, EndTag, StartTag, Tag, TokenSink, TokenSinkResult}; + +use crate::util::str::is_ascii_whitespace; + +use std::borrow::Cow::Borrowed; +use std::collections::VecDeque; +use std::default::Default; +use std::iter::{Enumerate, Rev}; +use std::mem::replace; +use std::{fmt, slice}; + +use crate::tokenizer::states::{RawData, RawKind}; +use crate::tree_builder::tag_sets::*; +use crate::tree_builder::types::*; +use crate::util::str::to_escaped_string; +use log::{debug, log_enabled, warn, Level}; +use mac::{_tt_as_expr_hack, format_if, matches}; + +pub use self::PushFlag::*; + +#[macro_use] +mod tag_sets; + +mod data; +mod types; + +include!(concat!(env!("OUT_DIR"), "/rules.rs")); + +/// Tree builder options, with an impl for Default. +#[derive(Copy, Clone)] +pub struct TreeBuilderOpts { + /// Report all parse errors described in the spec, at some + /// performance penalty? Default: false + pub exact_errors: bool, + + /// Is scripting enabled? + pub scripting_enabled: bool, + + /// Is this an `iframe srcdoc` document? + pub iframe_srcdoc: bool, + + /// Should we drop the DOCTYPE (if any) from the tree? + pub drop_doctype: bool, + + /// Obsolete, ignored. + pub ignore_missing_rules: bool, + + /// Initial TreeBuilder quirks mode. Default: NoQuirks + pub quirks_mode: QuirksMode, +} + +impl Default for TreeBuilderOpts { + fn default() -> TreeBuilderOpts { + TreeBuilderOpts { + exact_errors: false, + scripting_enabled: true, + iframe_srcdoc: false, + drop_doctype: false, + ignore_missing_rules: false, + quirks_mode: NoQuirks, + } + } +} + +/// The HTML tree builder. +pub struct TreeBuilder<Handle, Sink> { + /// Options controlling the behavior of the tree builder. + opts: TreeBuilderOpts, + + /// Consumer of tree modifications. + pub sink: Sink, + + /// Insertion mode. + mode: InsertionMode, + + /// Original insertion mode, used by Text and InTableText modes. + orig_mode: Option<InsertionMode>, + + /// Stack of template insertion modes. + template_modes: Vec<InsertionMode>, + + /// Pending table character tokens. + pending_table_text: Vec<(SplitStatus, StrTendril)>, + + /// Quirks mode as set by the parser. + /// FIXME: can scripts etc. change this? + quirks_mode: QuirksMode, + + /// The document node, which is created by the sink. + doc_handle: Handle, + + /// Stack of open elements, most recently added at end. + open_elems: Vec<Handle>, + + /// List of active formatting elements. + active_formatting: Vec<FormatEntry<Handle>>, + + //§ the-element-pointers + /// Head element pointer. + head_elem: Option<Handle>, + + /// Form element pointer. + form_elem: Option<Handle>, + //§ END + /// Frameset-ok flag. + frameset_ok: bool, + + /// Ignore a following U+000A LINE FEED? + ignore_lf: bool, + + /// Is foster parenting enabled? + foster_parenting: bool, + + /// The context element for the fragment parsing algorithm. + context_elem: Option<Handle>, + + /// Track current line + current_line: u64, + // WARNING: If you add new fields that contain Handles, you + // must add them to trace_handles() below to preserve memory + // safety! + // + // FIXME: Auto-generate the trace hooks like Servo does. +} + +impl<Handle, Sink> TreeBuilder<Handle, Sink> +where + Handle: Clone, + Sink: TreeSink<Handle = Handle>, +{ + /// Create a new tree builder which sends tree modifications to a particular `TreeSink`. + /// + /// The tree builder is also a `TokenSink`. + pub fn new(mut sink: Sink, opts: TreeBuilderOpts) -> TreeBuilder<Handle, Sink> { + let doc_handle = sink.get_document(); + TreeBuilder { + opts: opts, + sink: sink, + mode: Initial, + orig_mode: None, + template_modes: vec![], + pending_table_text: vec![], + quirks_mode: opts.quirks_mode, + doc_handle: doc_handle, + open_elems: vec![], + active_formatting: vec![], + head_elem: None, + form_elem: None, + frameset_ok: true, + ignore_lf: false, + foster_parenting: false, + context_elem: None, + current_line: 1, + } + } + + /// Create a new tree builder which sends tree modifications to a particular `TreeSink`. + /// This is for parsing fragments. + /// + /// The tree builder is also a `TokenSink`. + pub fn new_for_fragment( + mut sink: Sink, + context_elem: Handle, + form_elem: Option<Handle>, + opts: TreeBuilderOpts, + ) -> TreeBuilder<Handle, Sink> { + let doc_handle = sink.get_document(); + let context_is_template = sink.elem_name(&context_elem) == expanded_name!(html "template"); + let mut tb = TreeBuilder { + opts: opts, + sink: sink, + mode: Initial, + orig_mode: None, + template_modes: if context_is_template { + vec![InTemplate] + } else { + vec![] + }, + pending_table_text: vec![], + quirks_mode: opts.quirks_mode, + doc_handle: doc_handle, + open_elems: vec![], + active_formatting: vec![], + head_elem: None, + form_elem: form_elem, + frameset_ok: true, + ignore_lf: false, + foster_parenting: false, + context_elem: Some(context_elem), + current_line: 1, + }; + + // https://html.spec.whatwg.org/multipage/#parsing-html-fragments + // 5. Let root be a new html element with no attributes. + // 6. Append the element root to the Document node created above. + // 7. Set up the parser's stack of open elements so that it contains just the single element root. + tb.create_root(vec![]); + // 10. Reset the parser's insertion mode appropriately. + tb.mode = tb.reset_insertion_mode(); + + tb + } + + // https://html.spec.whatwg.org/multipage/#concept-frag-parse-context + // Step 4. Set the state of the HTML parser's tokenization stage as follows: + pub fn tokenizer_state_for_context_elem(&self) -> tok_state::State { + let elem = self.context_elem.as_ref().expect("no context element"); + let name = match self.sink.elem_name(elem) { + ExpandedName { + ns: &ns!(html), + local, + } => local, + _ => return tok_state::Data, + }; + match *name { + local_name!("title") | local_name!("textarea") => tok_state::RawData(tok_state::Rcdata), + + local_name!("style") | + local_name!("xmp") | + local_name!("iframe") | + local_name!("noembed") | + local_name!("noframes") => tok_state::RawData(tok_state::Rawtext), + + local_name!("script") => tok_state::RawData(tok_state::ScriptData), + + local_name!("noscript") => { + if self.opts.scripting_enabled { + tok_state::RawData(tok_state::Rawtext) + } else { + tok_state::Data + } + }, + + local_name!("plaintext") => tok_state::Plaintext, + + _ => tok_state::Data, + } + } + + /// Call the `Tracer`'s `trace_handle` method on every `Handle` in the tree builder's + /// internal state. This is intended to support garbage-collected DOMs. + pub fn trace_handles(&self, tracer: &Tracer<Handle = Handle>) { + tracer.trace_handle(&self.doc_handle); + for e in &self.open_elems { + tracer.trace_handle(e); + } + for e in &self.active_formatting { + match e { + &Element(ref h, _) => tracer.trace_handle(h), + _ => (), + } + } + self.head_elem.as_ref().map(|h| tracer.trace_handle(h)); + self.form_elem.as_ref().map(|h| tracer.trace_handle(h)); + self.context_elem.as_ref().map(|h| tracer.trace_handle(h)); + } + + #[allow(dead_code)] + fn dump_state(&self, label: String) { + println!("dump_state on {}", label); + print!(" open_elems:"); + for node in self.open_elems.iter() { + let name = self.sink.elem_name(node); + match *name.ns { + ns!(html) => print!(" {}", name.local), + _ => panic!(), + } + } + println!(""); + print!(" active_formatting:"); + for entry in self.active_formatting.iter() { + match entry { + &Marker => print!(" Marker"), + &Element(ref h, _) => { + let name = self.sink.elem_name(h); + match *name.ns { + ns!(html) => print!(" {}", name.local), + _ => panic!(), + } + }, + } + } + println!(""); + } + + fn debug_step(&self, mode: InsertionMode, token: &Token) { + if log_enabled!(Level::Debug) { + debug!( + "processing {} in insertion mode {:?}", + to_escaped_string(token), + mode + ); + } + } + + fn process_to_completion(&mut self, mut token: Token) -> TokenSinkResult<Handle> { + // Queue of additional tokens yet to be processed. + // This stays empty in the common case where we don't split whitespace. + let mut more_tokens = VecDeque::new(); + + loop { + let should_have_acknowledged_self_closing_flag = matches!( + token, + TagToken(Tag { + self_closing: true, + kind: StartTag, + .. + }) + ); + let result = if self.is_foreign(&token) { + self.step_foreign(token) + } else { + let mode = self.mode; + self.step(mode, token) + }; + match result { + Done => { + if should_have_acknowledged_self_closing_flag { + self.sink + .parse_error(Borrowed("Unacknowledged self-closing tag")); + } + token = unwrap_or_return!( + more_tokens.pop_front(), + tokenizer::TokenSinkResult::Continue + ); + }, + DoneAckSelfClosing => { + token = unwrap_or_return!( + more_tokens.pop_front(), + tokenizer::TokenSinkResult::Continue + ); + }, + Reprocess(m, t) => { + self.mode = m; + token = t; + }, + ReprocessForeign(t) => { + token = t; + }, + SplitWhitespace(mut buf) => { + let p = buf.pop_front_char_run(is_ascii_whitespace); + let (first, is_ws) = unwrap_or_return!(p, tokenizer::TokenSinkResult::Continue); + let status = if is_ws { Whitespace } else { NotWhitespace }; + token = CharacterTokens(status, first); + + if buf.len32() > 0 { + more_tokens.push_back(CharacterTokens(NotSplit, buf)); + } + }, + Script(node) => { + assert!(more_tokens.is_empty()); + return tokenizer::TokenSinkResult::Script(node); + }, + ToPlaintext => { + assert!(more_tokens.is_empty()); + return tokenizer::TokenSinkResult::Plaintext; + }, + ToRawData(k) => { + assert!(more_tokens.is_empty()); + return tokenizer::TokenSinkResult::RawData(k); + }, + } + } + } + + /// Are we parsing a HTML fragment? + pub fn is_fragment(&self) -> bool { + self.context_elem.is_some() + } + + /// https://html.spec.whatwg.org/multipage/#appropriate-place-for-inserting-a-node + fn appropriate_place_for_insertion( + &mut self, + override_target: Option<Handle>, + ) -> InsertionPoint<Handle> { + use self::tag_sets::*; + + declare_tag_set!(foster_target = "table" "tbody" "tfoot" "thead" "tr"); + let target = override_target.unwrap_or_else(|| self.current_node().clone()); + if !(self.foster_parenting && self.elem_in(&target, foster_target)) { + if self.html_elem_named(&target, local_name!("template")) { + // No foster parenting (inside template). + let contents = self.sink.get_template_contents(&target); + return LastChild(contents); + } else { + // No foster parenting (the common case). + return LastChild(target); + } + } + + // Foster parenting + let mut iter = self.open_elems.iter().rev().peekable(); + while let Some(elem) = iter.next() { + if self.html_elem_named(&elem, local_name!("template")) { + let contents = self.sink.get_template_contents(&elem); + return LastChild(contents); + } else if self.html_elem_named(&elem, local_name!("table")) { + return TableFosterParenting { + element: elem.clone(), + prev_element: (*iter.peek().unwrap()).clone(), + }; + } + } + let html_elem = self.html_elem(); + LastChild(html_elem.clone()) + } + + fn insert_at(&mut self, insertion_point: InsertionPoint<Handle>, child: NodeOrText<Handle>) { + match insertion_point { + LastChild(parent) => self.sink.append(&parent, child), + BeforeSibling(sibling) => self.sink.append_before_sibling(&sibling, child), + TableFosterParenting { + element, + prev_element, + } => self + .sink + .append_based_on_parent_node(&element, &prev_element, child), + } + } +} + +impl<Handle, Sink> TokenSink for TreeBuilder<Handle, Sink> +where + Handle: Clone, + Sink: TreeSink<Handle = Handle>, +{ + type Handle = Handle; + + fn process_token( + &mut self, + token: tokenizer::Token, + line_number: u64, + ) -> TokenSinkResult<Handle> { + if line_number != self.current_line { + self.sink.set_current_line(line_number); + } + let ignore_lf = replace(&mut self.ignore_lf, false); + + // Handle `ParseError` and `DoctypeToken`; convert everything else to the local `Token` type. + let token = match token { + tokenizer::ParseError(e) => { + self.sink.parse_error(e); + return tokenizer::TokenSinkResult::Continue; + }, + + tokenizer::DoctypeToken(dt) => { + if self.mode == Initial { + let (err, quirk) = data::doctype_error_and_quirks(&dt, self.opts.iframe_srcdoc); + if err { + self.sink.parse_error(format_if!( + self.opts.exact_errors, + "Bad DOCTYPE", + "Bad DOCTYPE: {:?}", + dt + )); + } + let Doctype { + name, + public_id, + system_id, + force_quirks: _, + } = dt; + if !self.opts.drop_doctype { + self.sink.append_doctype_to_document( + name.unwrap_or(StrTendril::new()), + public_id.unwrap_or(StrTendril::new()), + system_id.unwrap_or(StrTendril::new()), + ); + } + self.set_quirks_mode(quirk); + + self.mode = BeforeHtml; + return tokenizer::TokenSinkResult::Continue; + } else { + self.sink.parse_error(format_if!( + self.opts.exact_errors, + "DOCTYPE in body", + "DOCTYPE in insertion mode {:?}", + self.mode + )); + return tokenizer::TokenSinkResult::Continue; + } + }, + + tokenizer::TagToken(x) => TagToken(x), + tokenizer::CommentToken(x) => CommentToken(x), + tokenizer::NullCharacterToken => NullCharacterToken, + tokenizer::EOFToken => EOFToken, + + tokenizer::CharacterTokens(mut x) => { + if ignore_lf && x.starts_with("\n") { + x.pop_front(1); + } + if x.is_empty() { + return tokenizer::TokenSinkResult::Continue; + } + CharacterTokens(NotSplit, x) + }, + }; + + self.process_to_completion(token) + } + + fn end(&mut self) { + for elem in self.open_elems.drain(..).rev() { + self.sink.pop(&elem); + } + } + + fn adjusted_current_node_present_but_not_in_html_namespace(&self) -> bool { + !self.open_elems.is_empty() && + self.sink.elem_name(self.adjusted_current_node()).ns != &ns!(html) + } +} + +pub fn html_elem<Handle>(open_elems: &[Handle]) -> &Handle { + &open_elems[0] +} + +pub struct ActiveFormattingIter<'a, Handle: 'a> { + iter: Rev<Enumerate<slice::Iter<'a, FormatEntry<Handle>>>>, +} + +impl<'a, Handle> Iterator for ActiveFormattingIter<'a, Handle> { + type Item = (usize, &'a Handle, &'a Tag); + fn next(&mut self) -> Option<(usize, &'a Handle, &'a Tag)> { + match self.iter.next() { + None | Some((_, &Marker)) => None, + Some((i, &Element(ref h, ref t))) => Some((i, h, t)), + } + } +} + +pub enum PushFlag { + Push, + NoPush, +} + +enum Bookmark<Handle> { + Replace(Handle), + InsertAfter(Handle), +} + +macro_rules! qualname { + ("", $local:tt) => { + QualName { + prefix: None, + ns: ns!(), + local: local_name!($local), + } + }; + ($prefix: tt $ns:tt $local:tt) => { + QualName { + prefix: Some(namespace_prefix!($prefix)), + ns: ns!($ns), + local: local_name!($local), + } + }; +} + +#[doc(hidden)] +impl<Handle, Sink> TreeBuilder<Handle, Sink> +where + Handle: Clone, + Sink: TreeSink<Handle = Handle>, +{ + fn unexpected<T: fmt::Debug>(&mut self, _thing: &T) -> ProcessResult<Handle> { + self.sink.parse_error(format_if!( + self.opts.exact_errors, + "Unexpected token", + "Unexpected token {} in insertion mode {:?}", + to_escaped_string(_thing), + self.mode + )); + Done + } + + fn assert_named(&mut self, node: &Handle, name: LocalName) { + assert!(self.html_elem_named(&node, name)); + } + + /// Iterate over the active formatting elements (with index in the list) from the end + /// to the last marker, or the beginning if there are no markers. + fn active_formatting_end_to_marker<'a>(&'a self) -> ActiveFormattingIter<'a, Handle> { + ActiveFormattingIter { + iter: self.active_formatting.iter().enumerate().rev(), + } + } + + fn position_in_active_formatting(&self, element: &Handle) -> Option<usize> { + self.active_formatting.iter().position(|n| match n { + &Marker => false, + &Element(ref handle, _) => self.sink.same_node(handle, element), + }) + } + + fn set_quirks_mode(&mut self, mode: QuirksMode) { + self.quirks_mode = mode; + self.sink.set_quirks_mode(mode); + } + + fn stop_parsing(&mut self) -> ProcessResult<Handle> { + Done + } + + //§ parsing-elements-that-contain-only-text + // Switch to `Text` insertion mode, save the old mode, and + // switch the tokenizer to a raw-data state. + // The latter only takes effect after the current / next + // `process_token` of a start tag returns! + fn to_raw_text_mode(&mut self, k: RawKind) -> ProcessResult<Handle> { + self.orig_mode = Some(self.mode); + self.mode = Text; + ToRawData(k) + } + + // The generic raw text / RCDATA parsing algorithm. + fn parse_raw_data(&mut self, tag: Tag, k: RawKind) -> ProcessResult<Handle> { + self.insert_element_for(tag); + self.to_raw_text_mode(k) + } + //§ END + + fn current_node(&self) -> &Handle { + self.open_elems.last().expect("no current element") + } + + fn adjusted_current_node(&self) -> &Handle { + if self.open_elems.len() == 1 { + if let Some(ctx) = self.context_elem.as_ref() { + return ctx; + } + } + self.current_node() + } + + fn current_node_in<TagSet>(&self, set: TagSet) -> bool + where + TagSet: Fn(ExpandedName) -> bool, + { + set(self.sink.elem_name(self.current_node())) + } + + // Insert at the "appropriate place for inserting a node". + fn insert_appropriately(&mut self, child: NodeOrText<Handle>, override_target: Option<Handle>) { + let insertion_point = self.appropriate_place_for_insertion(override_target); + self.insert_at(insertion_point, child); + } + + fn adoption_agency(&mut self, subject: LocalName) { + // 1. + if self.current_node_named(subject.clone()) { + if self + .position_in_active_formatting(self.current_node()) + .is_none() + { + self.pop(); + return; + } + } + + // 2. 3. 4. + for _ in 0..8 { + // 5. + let (fmt_elem_index, fmt_elem, fmt_elem_tag) = unwrap_or_return!( + // We clone the Handle and Tag so they don't cause an immutable borrow of self. + self.active_formatting_end_to_marker() + .filter(|&(_, _, tag)| tag.name == subject) + .next() + .map(|(i, h, t)| (i, h.clone(), t.clone())), + { + self.process_end_tag_in_body(Tag { + kind: EndTag, + name: subject, + self_closing: false, + attrs: vec![], + }); + } + ); + + let fmt_elem_stack_index = unwrap_or_return!( + self.open_elems + .iter() + .rposition(|n| self.sink.same_node(n, &fmt_elem)), + { + self.sink + .parse_error(Borrowed("Formatting element not open")); + self.active_formatting.remove(fmt_elem_index); + } + ); + + // 7. + if !self.in_scope(default_scope, |n| self.sink.same_node(&n, &fmt_elem)) { + self.sink + .parse_error(Borrowed("Formatting element not in scope")); + return; + } + + // 8. + if !self.sink.same_node(self.current_node(), &fmt_elem) { + self.sink + .parse_error(Borrowed("Formatting element not current node")); + } + + // 9. + let (furthest_block_index, furthest_block) = unwrap_or_return!( + self.open_elems + .iter() + .enumerate() + .skip(fmt_elem_stack_index) + .filter(|&(_, open_element)| self.elem_in(open_element, special_tag)) + .next() + .map(|(i, h)| (i, h.clone())), + // 10. + { + self.open_elems.truncate(fmt_elem_stack_index); + self.active_formatting.remove(fmt_elem_index); + } + ); + + // 11. + let common_ancestor = self.open_elems[fmt_elem_stack_index - 1].clone(); + + // 12. + let mut bookmark = Bookmark::Replace(fmt_elem.clone()); + + // 13. + let mut node; + let mut node_index = furthest_block_index; + let mut last_node = furthest_block.clone(); + + // 13.1. + let mut inner_counter = 0; + loop { + // 13.2. + inner_counter += 1; + + // 13.3. + node_index -= 1; + node = self.open_elems[node_index].clone(); + + // 13.4. + if self.sink.same_node(&node, &fmt_elem) { + break; + } + + // 13.5. + if inner_counter > 3 { + self.position_in_active_formatting(&node) + .map(|position| self.active_formatting.remove(position)); + self.open_elems.remove(node_index); + continue; + } + + let node_formatting_index = unwrap_or_else!( + self.position_in_active_formatting(&node), + // 13.6. + { + self.open_elems.remove(node_index); + continue; + } + ); + + // 13.7. + let tag = match self.active_formatting[node_formatting_index] { + Element(ref h, ref t) => { + assert!(self.sink.same_node(h, &node)); + t.clone() + }, + Marker => panic!("Found marker during adoption agency"), + }; + // FIXME: Is there a way to avoid cloning the attributes twice here (once on their + // own, once as part of t.clone() above)? + let new_element = create_element( + &mut self.sink, + QualName::new(None, ns!(html), tag.name.clone()), + tag.attrs.clone(), + ); + self.open_elems[node_index] = new_element.clone(); + self.active_formatting[node_formatting_index] = Element(new_element.clone(), tag); + node = new_element; + + // 13.8. + if self.sink.same_node(&last_node, &furthest_block) { + bookmark = Bookmark::InsertAfter(node.clone()); + } + + // 13.9. + self.sink.remove_from_parent(&last_node); + self.sink.append(&node, AppendNode(last_node.clone())); + + // 13.10. + last_node = node.clone(); + + // 13.11. + } + + // 14. + self.sink.remove_from_parent(&last_node); + self.insert_appropriately(AppendNode(last_node.clone()), Some(common_ancestor)); + + // 15. + // FIXME: Is there a way to avoid cloning the attributes twice here (once on their own, + // once as part of t.clone() above)? + let new_element = create_element( + &mut self.sink, + QualName::new(None, ns!(html), fmt_elem_tag.name.clone()), + fmt_elem_tag.attrs.clone(), + ); + let new_entry = Element(new_element.clone(), fmt_elem_tag); + + // 16. + self.sink.reparent_children(&furthest_block, &new_element); + + // 17. + self.sink + .append(&furthest_block, AppendNode(new_element.clone())); + + // 18. + // FIXME: We could probably get rid of the position_in_active_formatting() calls here + // if we had a more clever Bookmark representation. + match bookmark { + Bookmark::Replace(to_replace) => { + let index = self + .position_in_active_formatting(&to_replace) + .expect("bookmark not found in active formatting elements"); + self.active_formatting[index] = new_entry; + }, + Bookmark::InsertAfter(previous) => { + let index = self + .position_in_active_formatting(&previous) + .expect("bookmark not found in active formatting elements") + + 1; + self.active_formatting.insert(index, new_entry); + let old_index = self + .position_in_active_formatting(&fmt_elem) + .expect("formatting element not found in active formatting elements"); + self.active_formatting.remove(old_index); + }, + } + + // 19. + self.remove_from_stack(&fmt_elem); + let new_furthest_block_index = self + .open_elems + .iter() + .position(|n| self.sink.same_node(n, &furthest_block)) + .expect("furthest block missing from open element stack"); + self.open_elems + .insert(new_furthest_block_index + 1, new_element); + + // 20. + } + } + + fn push(&mut self, elem: &Handle) { + self.open_elems.push(elem.clone()); + } + + fn pop(&mut self) -> Handle { + let elem = self.open_elems.pop().expect("no current element"); + self.sink.pop(&elem); + elem + } + + fn remove_from_stack(&mut self, elem: &Handle) { + let sink = &mut self.sink; + let position = self + .open_elems + .iter() + .rposition(|x| sink.same_node(elem, &x)); + if let Some(position) = position { + self.open_elems.remove(position); + sink.pop(elem); + } + } + + fn is_marker_or_open(&self, entry: &FormatEntry<Handle>) -> bool { + match *entry { + Marker => true, + Element(ref node, _) => self + .open_elems + .iter() + .rev() + .any(|n| self.sink.same_node(&n, &node)), + } + } + + /// Reconstruct the active formatting elements. + fn reconstruct_formatting(&mut self) { + { + let last = unwrap_or_return!(self.active_formatting.last(), ()); + if self.is_marker_or_open(last) { + return; + } + } + + let mut entry_index = self.active_formatting.len() - 1; + loop { + if entry_index == 0 { + break; + } + entry_index -= 1; + if self.is_marker_or_open(&self.active_formatting[entry_index]) { + entry_index += 1; + break; + } + } + + loop { + let tag = match self.active_formatting[entry_index] { + Element(_, ref t) => t.clone(), + Marker => panic!("Found marker during formatting element reconstruction"), + }; + + // FIXME: Is there a way to avoid cloning the attributes twice here (once on their own, + // once as part of t.clone() above)? + let new_element = + self.insert_element(Push, ns!(html), tag.name.clone(), tag.attrs.clone()); + self.active_formatting[entry_index] = Element(new_element, tag); + if entry_index == self.active_formatting.len() - 1 { + break; + } + entry_index += 1; + } + } + + /// Get the first element on the stack, which will be the <html> element. + fn html_elem(&self) -> &Handle { + &self.open_elems[0] + } + + /// Get the second element on the stack, if it's a HTML body element. + fn body_elem(&self) -> Option<&Handle> { + if self.open_elems.len() <= 1 { + return None; + } + + let node = &self.open_elems[1]; + if self.html_elem_named(node, local_name!("body")) { + Some(node) + } else { + None + } + } + + /// Signal an error depending on the state of the stack of open elements at + /// the end of the body. + fn check_body_end(&mut self) { + declare_tag_set!(body_end_ok = + "dd" "dt" "li" "optgroup" "option" "p" "rp" "rt" "tbody" "td" "tfoot" "th" + "thead" "tr" "body" "html"); + + for elem in self.open_elems.iter() { + let error; + { + let name = self.sink.elem_name(elem); + if body_end_ok(name) { + continue; + } + error = format_if!( + self.opts.exact_errors, + "Unexpected open tag at end of body", + "Unexpected open tag {:?} at end of body", + name + ); + } + self.sink.parse_error(error); + // FIXME: Do we keep checking after finding one bad tag? + // The spec suggests not. + return; + } + } + + fn in_scope<TagSet, Pred>(&self, scope: TagSet, pred: Pred) -> bool + where + TagSet: Fn(ExpandedName) -> bool, + Pred: Fn(Handle) -> bool, + { + for node in self.open_elems.iter().rev() { + if pred(node.clone()) { + return true; + } + if scope(self.sink.elem_name(node)) { + return false; + } + } + + // supposed to be impossible, because <html> is always in scope + + false + } + + fn elem_in<TagSet>(&self, elem: &Handle, set: TagSet) -> bool + where + TagSet: Fn(ExpandedName) -> bool, + { + set(self.sink.elem_name(elem)) + } + + fn html_elem_named(&self, elem: &Handle, name: LocalName) -> bool { + let expanded = self.sink.elem_name(elem); + *expanded.ns == ns!(html) && *expanded.local == name + } + + fn in_html_elem_named(&self, name: LocalName) -> bool { + self.open_elems + .iter() + .any(|elem| self.html_elem_named(elem, name.clone())) + } + + fn current_node_named(&self, name: LocalName) -> bool { + self.html_elem_named(self.current_node(), name) + } + + fn in_scope_named<TagSet>(&self, scope: TagSet, name: LocalName) -> bool + where + TagSet: Fn(ExpandedName) -> bool, + { + self.in_scope(scope, |elem| self.html_elem_named(&elem, name.clone())) + } + + //§ closing-elements-that-have-implied-end-tags + fn generate_implied_end<TagSet>(&mut self, set: TagSet) + where + TagSet: Fn(ExpandedName) -> bool, + { + loop { + { + let elem = unwrap_or_return!(self.open_elems.last(), ()); + let nsname = self.sink.elem_name(elem); + if !set(nsname) { + return; + } + } + self.pop(); + } + } + + fn generate_implied_end_except(&mut self, except: LocalName) { + self.generate_implied_end(|p| { + if *p.ns == ns!(html) && *p.local == except { + false + } else { + cursory_implied_end(p) + } + }); + } + //§ END + + // Pop elements until the current element is in the set. + fn pop_until_current<TagSet>(&mut self, pred: TagSet) + where + TagSet: Fn(ExpandedName) -> bool, + { + loop { + if self.current_node_in(|x| pred(x)) { + break; + } + self.open_elems.pop(); + } + } + + // Pop elements until an element from the set has been popped. Returns the + // number of elements popped. + fn pop_until<P>(&mut self, pred: P) -> usize + where + P: Fn(ExpandedName) -> bool, + { + let mut n = 0; + loop { + n += 1; + match self.open_elems.pop() { + None => break, + Some(elem) => { + if pred(self.sink.elem_name(&elem)) { + break; + } + }, + } + } + n + } + + fn pop_until_named(&mut self, name: LocalName) -> usize { + self.pop_until(|p| *p.ns == ns!(html) && *p.local == name) + } + + // Pop elements until one with the specified name has been popped. + // Signal an error if it was not the first one. + fn expect_to_close(&mut self, name: LocalName) { + if self.pop_until_named(name.clone()) != 1 { + self.sink.parse_error(format_if!( + self.opts.exact_errors, + "Unexpected open element", + "Unexpected open element while closing {:?}", + name + )); + } + } + + fn close_p_element(&mut self) { + declare_tag_set!(implied = [cursory_implied_end] - "p"); + self.generate_implied_end(implied); + self.expect_to_close(local_name!("p")); + } + + fn close_p_element_in_button_scope(&mut self) { + if self.in_scope_named(button_scope, local_name!("p")) { + self.close_p_element(); + } + } + + // Check <input> tags for type=hidden + fn is_type_hidden(&self, tag: &Tag) -> bool { + match tag + .attrs + .iter() + .find(|&at| at.name.expanded() == expanded_name!("", "type")) + { + None => false, + Some(at) => (&*at.value).eq_ignore_ascii_case("hidden"), + } + } + + fn foster_parent_in_body(&mut self, token: Token) -> ProcessResult<Handle> { + warn!("foster parenting not implemented"); + self.foster_parenting = true; + let res = self.step(InBody, token); + // FIXME: what if res is Reprocess? + self.foster_parenting = false; + res + } + + fn process_chars_in_table(&mut self, token: Token) -> ProcessResult<Handle> { + declare_tag_set!(table_outer = "table" "tbody" "tfoot" "thead" "tr"); + if self.current_node_in(table_outer) { + assert!(self.pending_table_text.is_empty()); + self.orig_mode = Some(self.mode); + Reprocess(InTableText, token) + } else { + self.sink.parse_error(format_if!( + self.opts.exact_errors, + "Unexpected characters in table", + "Unexpected characters {} in table", + to_escaped_string(&token) + )); + self.foster_parent_in_body(token) + } + } + + // https://html.spec.whatwg.org/multipage/#reset-the-insertion-mode-appropriately + fn reset_insertion_mode(&mut self) -> InsertionMode { + for (i, mut node) in self.open_elems.iter().enumerate().rev() { + let last = i == 0usize; + if let (true, Some(ctx)) = (last, self.context_elem.as_ref()) { + node = ctx; + } + let name = match self.sink.elem_name(node) { + ExpandedName { + ns: &ns!(html), + local, + } => local, + _ => continue, + }; + match *name { + local_name!("select") => { + for ancestor in self.open_elems[0..i].iter().rev() { + if self.html_elem_named(ancestor, local_name!("template")) { + return InSelect; + } else if self.html_elem_named(ancestor, local_name!("table")) { + return InSelectInTable; + } + } + return InSelect; + }, + local_name!("td") | local_name!("th") => { + if !last { + return InCell; + } + }, + local_name!("tr") => return InRow, + local_name!("tbody") | local_name!("thead") | local_name!("tfoot") => { + return InTableBody; + }, + local_name!("caption") => return InCaption, + local_name!("colgroup") => return InColumnGroup, + local_name!("table") => return InTable, + local_name!("template") => return *self.template_modes.last().unwrap(), + local_name!("head") => { + if !last { + return InHead; + } + }, + local_name!("body") => return InBody, + local_name!("frameset") => return InFrameset, + local_name!("html") => match self.head_elem { + None => return BeforeHead, + Some(_) => return AfterHead, + }, + + _ => (), + } + } + InBody + } + + fn close_the_cell(&mut self) { + self.generate_implied_end(cursory_implied_end); + if self.pop_until(td_th) != 1 { + self.sink + .parse_error(Borrowed("expected to close <td> or <th> with cell")); + } + self.clear_active_formatting_to_marker(); + } + + fn append_text(&mut self, text: StrTendril) -> ProcessResult<Handle> { + self.insert_appropriately(AppendText(text), None); + Done + } + + fn append_comment(&mut self, text: StrTendril) -> ProcessResult<Handle> { + let comment = self.sink.create_comment(text); + self.insert_appropriately(AppendNode(comment), None); + Done + } + + fn append_comment_to_doc(&mut self, text: StrTendril) -> ProcessResult<Handle> { + let comment = self.sink.create_comment(text); + self.sink.append(&self.doc_handle, AppendNode(comment)); + Done + } + + fn append_comment_to_html(&mut self, text: StrTendril) -> ProcessResult<Handle> { + let target = html_elem(&self.open_elems); + let comment = self.sink.create_comment(text); + self.sink.append(target, AppendNode(comment)); + Done + } + + //§ creating-and-inserting-nodes + fn create_root(&mut self, attrs: Vec<Attribute>) { + let elem = create_element( + &mut self.sink, + QualName::new(None, ns!(html), local_name!("html")), + attrs, + ); + self.push(&elem); + self.sink.append(&self.doc_handle, AppendNode(elem)); + // FIXME: application cache selection algorithm + } + + // https://html.spec.whatwg.org/multipage/#create-an-element-for-the-token + fn insert_element( + &mut self, + push: PushFlag, + ns: Namespace, + name: LocalName, + attrs: Vec<Attribute>, + ) -> Handle { + declare_tag_set!(form_associatable = + "button" "fieldset" "input" "object" + "output" "select" "textarea" "img"); + + declare_tag_set!(listed = [form_associatable] - "img"); + + // Step 7. + let qname = QualName::new(None, ns, name); + let elem = create_element(&mut self.sink, qname.clone(), attrs.clone()); + + let insertion_point = self.appropriate_place_for_insertion(None); + let (node1, node2) = match insertion_point { + LastChild(ref p) | BeforeSibling(ref p) => (p.clone(), None), + TableFosterParenting { + ref element, + ref prev_element, + } => (element.clone(), Some(prev_element.clone())), + }; + + // Step 12. + if form_associatable(qname.expanded()) && + self.form_elem.is_some() && + !self.in_html_elem_named(local_name!("template")) && + !(listed(qname.expanded()) && + attrs + .iter() + .any(|a| a.name.expanded() == expanded_name!("", "form"))) + { + let form = self.form_elem.as_ref().unwrap().clone(); + let node2 = match node2 { + Some(ref n) => Some(n), + None => None, + }; + self.sink.associate_with_form(&elem, &form, (&node1, node2)); + } + + self.insert_at(insertion_point, AppendNode(elem.clone())); + + match push { + Push => self.push(&elem), + NoPush => (), + } + // FIXME: Remove from the stack if we can't append? + elem + } + + fn insert_element_for(&mut self, tag: Tag) -> Handle { + self.insert_element(Push, ns!(html), tag.name, tag.attrs) + } + + fn insert_and_pop_element_for(&mut self, tag: Tag) -> Handle { + self.insert_element(NoPush, ns!(html), tag.name, tag.attrs) + } + + fn insert_phantom(&mut self, name: LocalName) -> Handle { + self.insert_element(Push, ns!(html), name, vec![]) + } + //§ END + + fn create_formatting_element_for(&mut self, tag: Tag) -> Handle { + // FIXME: This really wants unit tests. + let mut first_match = None; + let mut matches = 0usize; + for (i, _, old_tag) in self.active_formatting_end_to_marker() { + if tag.equiv_modulo_attr_order(old_tag) { + first_match = Some(i); + matches += 1; + } + } + + if matches >= 3 { + self.active_formatting + .remove(first_match.expect("matches with no index")); + } + + let elem = self.insert_element(Push, ns!(html), tag.name.clone(), tag.attrs.clone()); + self.active_formatting.push(Element(elem.clone(), tag)); + elem + } + + fn clear_active_formatting_to_marker(&mut self) { + loop { + match self.active_formatting.pop() { + None | Some(Marker) => break, + _ => (), + } + } + } + + fn process_end_tag_in_body(&mut self, tag: Tag) { + // Look back for a matching open element. + let mut match_idx = None; + for (i, elem) in self.open_elems.iter().enumerate().rev() { + if self.html_elem_named(elem, tag.name.clone()) { + match_idx = Some(i); + break; + } + + if self.elem_in(elem, special_tag) { + self.sink + .parse_error(Borrowed("Found special tag while closing generic tag")); + return; + } + } + + // Can't use unwrap_or_return!() due to rust-lang/rust#16617. + let match_idx = match match_idx { + None => { + // I believe this is impossible, because the root + // <html> element is in special_tag. + self.unexpected(&tag); + return; + }, + Some(x) => x, + }; + + self.generate_implied_end_except(tag.name.clone()); + + if match_idx != self.open_elems.len() - 1 { + // mis-nested tags + self.unexpected(&tag); + } + self.open_elems.truncate(match_idx); + } + + fn handle_misnested_a_tags(&mut self, tag: &Tag) { + let node = unwrap_or_return!( + self.active_formatting_end_to_marker() + .filter(|&(_, n, _)| self.html_elem_named(n, local_name!("a"))) + .next() + .map(|(_, n, _)| n.clone()), + () + ); + + self.unexpected(tag); + self.adoption_agency(local_name!("a")); + self.position_in_active_formatting(&node) + .map(|index| self.active_formatting.remove(index)); + self.remove_from_stack(&node); + } + + //§ tree-construction + fn is_foreign(&mut self, token: &Token) -> bool { + if let EOFToken = *token { + return false; + } + + if self.open_elems.is_empty() { + return false; + } + + let name = self.sink.elem_name(self.adjusted_current_node()); + if let ns!(html) = *name.ns { + return false; + } + + if mathml_text_integration_point(name) { + match *token { + CharacterTokens(..) | NullCharacterToken => return false, + TagToken(Tag { + kind: StartTag, + ref name, + .. + }) if !matches!(*name, local_name!("mglyph") | local_name!("malignmark")) => { + return false; + }, + _ => (), + } + } + + if svg_html_integration_point(name) { + match *token { + CharacterTokens(..) | NullCharacterToken => return false, + TagToken(Tag { kind: StartTag, .. }) => return false, + _ => (), + } + } + + if let expanded_name!(mathml "annotation-xml") = name { + match *token { + TagToken(Tag { + kind: StartTag, + name: local_name!("svg"), + .. + }) => return false, + CharacterTokens(..) | NullCharacterToken | TagToken(Tag { kind: StartTag, .. }) => { + return !self + .sink + .is_mathml_annotation_xml_integration_point(self.adjusted_current_node()); + }, + _ => {}, + }; + } + + true + } + //§ END + + fn enter_foreign(&mut self, mut tag: Tag, ns: Namespace) -> ProcessResult<Handle> { + match ns { + ns!(mathml) => self.adjust_mathml_attributes(&mut tag), + ns!(svg) => self.adjust_svg_attributes(&mut tag), + _ => (), + } + self.adjust_foreign_attributes(&mut tag); + + if tag.self_closing { + self.insert_element(NoPush, ns, tag.name, tag.attrs); + DoneAckSelfClosing + } else { + self.insert_element(Push, ns, tag.name, tag.attrs); + Done + } + } + + fn adjust_svg_tag_name(&mut self, tag: &mut Tag) { + let Tag { ref mut name, .. } = *tag; + match *name { + local_name!("altglyph") => *name = local_name!("altGlyph"), + local_name!("altglyphdef") => *name = local_name!("altGlyphDef"), + local_name!("altglyphitem") => *name = local_name!("altGlyphItem"), + local_name!("animatecolor") => *name = local_name!("animateColor"), + local_name!("animatemotion") => *name = local_name!("animateMotion"), + local_name!("animatetransform") => *name = local_name!("animateTransform"), + local_name!("clippath") => *name = local_name!("clipPath"), + local_name!("feblend") => *name = local_name!("feBlend"), + local_name!("fecolormatrix") => *name = local_name!("feColorMatrix"), + local_name!("fecomponenttransfer") => *name = local_name!("feComponentTransfer"), + local_name!("fecomposite") => *name = local_name!("feComposite"), + local_name!("feconvolvematrix") => *name = local_name!("feConvolveMatrix"), + local_name!("fediffuselighting") => *name = local_name!("feDiffuseLighting"), + local_name!("fedisplacementmap") => *name = local_name!("feDisplacementMap"), + local_name!("fedistantlight") => *name = local_name!("feDistantLight"), + local_name!("fedropshadow") => *name = local_name!("feDropShadow"), + local_name!("feflood") => *name = local_name!("feFlood"), + local_name!("fefunca") => *name = local_name!("feFuncA"), + local_name!("fefuncb") => *name = local_name!("feFuncB"), + local_name!("fefuncg") => *name = local_name!("feFuncG"), + local_name!("fefuncr") => *name = local_name!("feFuncR"), + local_name!("fegaussianblur") => *name = local_name!("feGaussianBlur"), + local_name!("feimage") => *name = local_name!("feImage"), + local_name!("femerge") => *name = local_name!("feMerge"), + local_name!("femergenode") => *name = local_name!("feMergeNode"), + local_name!("femorphology") => *name = local_name!("feMorphology"), + local_name!("feoffset") => *name = local_name!("feOffset"), + local_name!("fepointlight") => *name = local_name!("fePointLight"), + local_name!("fespecularlighting") => *name = local_name!("feSpecularLighting"), + local_name!("fespotlight") => *name = local_name!("feSpotLight"), + local_name!("fetile") => *name = local_name!("feTile"), + local_name!("feturbulence") => *name = local_name!("feTurbulence"), + local_name!("foreignobject") => *name = local_name!("foreignObject"), + local_name!("glyphref") => *name = local_name!("glyphRef"), + local_name!("lineargradient") => *name = local_name!("linearGradient"), + local_name!("radialgradient") => *name = local_name!("radialGradient"), + local_name!("textpath") => *name = local_name!("textPath"), + _ => (), + } + } + + fn adjust_attributes<F>(&mut self, tag: &mut Tag, mut map: F) + where + F: FnMut(LocalName) -> Option<QualName>, + { + for &mut Attribute { ref mut name, .. } in &mut tag.attrs { + if let Some(replacement) = map(name.local.clone()) { + *name = replacement; + } + } + } + + fn adjust_svg_attributes(&mut self, tag: &mut Tag) { + self.adjust_attributes(tag, |k| match k { + local_name!("attributename") => Some(qualname!("", "attributeName")), + local_name!("attributetype") => Some(qualname!("", "attributeType")), + local_name!("basefrequency") => Some(qualname!("", "baseFrequency")), + local_name!("baseprofile") => Some(qualname!("", "baseProfile")), + local_name!("calcmode") => Some(qualname!("", "calcMode")), + local_name!("clippathunits") => Some(qualname!("", "clipPathUnits")), + local_name!("diffuseconstant") => Some(qualname!("", "diffuseConstant")), + local_name!("edgemode") => Some(qualname!("", "edgeMode")), + local_name!("filterunits") => Some(qualname!("", "filterUnits")), + local_name!("glyphref") => Some(qualname!("", "glyphRef")), + local_name!("gradienttransform") => Some(qualname!("", "gradientTransform")), + local_name!("gradientunits") => Some(qualname!("", "gradientUnits")), + local_name!("kernelmatrix") => Some(qualname!("", "kernelMatrix")), + local_name!("kernelunitlength") => Some(qualname!("", "kernelUnitLength")), + local_name!("keypoints") => Some(qualname!("", "keyPoints")), + local_name!("keysplines") => Some(qualname!("", "keySplines")), + local_name!("keytimes") => Some(qualname!("", "keyTimes")), + local_name!("lengthadjust") => Some(qualname!("", "lengthAdjust")), + local_name!("limitingconeangle") => Some(qualname!("", "limitingConeAngle")), + local_name!("markerheight") => Some(qualname!("", "markerHeight")), + local_name!("markerunits") => Some(qualname!("", "markerUnits")), + local_name!("markerwidth") => Some(qualname!("", "markerWidth")), + local_name!("maskcontentunits") => Some(qualname!("", "maskContentUnits")), + local_name!("maskunits") => Some(qualname!("", "maskUnits")), + local_name!("numoctaves") => Some(qualname!("", "numOctaves")), + local_name!("pathlength") => Some(qualname!("", "pathLength")), + local_name!("patterncontentunits") => Some(qualname!("", "patternContentUnits")), + local_name!("patterntransform") => Some(qualname!("", "patternTransform")), + local_name!("patternunits") => Some(qualname!("", "patternUnits")), + local_name!("pointsatx") => Some(qualname!("", "pointsAtX")), + local_name!("pointsaty") => Some(qualname!("", "pointsAtY")), + local_name!("pointsatz") => Some(qualname!("", "pointsAtZ")), + local_name!("preservealpha") => Some(qualname!("", "preserveAlpha")), + local_name!("preserveaspectratio") => Some(qualname!("", "preserveAspectRatio")), + local_name!("primitiveunits") => Some(qualname!("", "primitiveUnits")), + local_name!("refx") => Some(qualname!("", "refX")), + local_name!("refy") => Some(qualname!("", "refY")), + local_name!("repeatcount") => Some(qualname!("", "repeatCount")), + local_name!("repeatdur") => Some(qualname!("", "repeatDur")), + local_name!("requiredextensions") => Some(qualname!("", "requiredExtensions")), + local_name!("requiredfeatures") => Some(qualname!("", "requiredFeatures")), + local_name!("specularconstant") => Some(qualname!("", "specularConstant")), + local_name!("specularexponent") => Some(qualname!("", "specularExponent")), + local_name!("spreadmethod") => Some(qualname!("", "spreadMethod")), + local_name!("startoffset") => Some(qualname!("", "startOffset")), + local_name!("stddeviation") => Some(qualname!("", "stdDeviation")), + local_name!("stitchtiles") => Some(qualname!("", "stitchTiles")), + local_name!("surfacescale") => Some(qualname!("", "surfaceScale")), + local_name!("systemlanguage") => Some(qualname!("", "systemLanguage")), + local_name!("tablevalues") => Some(qualname!("", "tableValues")), + local_name!("targetx") => Some(qualname!("", "targetX")), + local_name!("targety") => Some(qualname!("", "targetY")), + local_name!("textlength") => Some(qualname!("", "textLength")), + local_name!("viewbox") => Some(qualname!("", "viewBox")), + local_name!("viewtarget") => Some(qualname!("", "viewTarget")), + local_name!("xchannelselector") => Some(qualname!("", "xChannelSelector")), + local_name!("ychannelselector") => Some(qualname!("", "yChannelSelector")), + local_name!("zoomandpan") => Some(qualname!("", "zoomAndPan")), + _ => None, + }); + } + + fn adjust_mathml_attributes(&mut self, tag: &mut Tag) { + self.adjust_attributes(tag, |k| match k { + local_name!("definitionurl") => Some(qualname!("", "definitionURL")), + _ => None, + }); + } + + fn adjust_foreign_attributes(&mut self, tag: &mut Tag) { + self.adjust_attributes(tag, |k| match k { + local_name!("xlink:actuate") => Some(qualname!("xlink" xlink "actuate")), + local_name!("xlink:arcrole") => Some(qualname!("xlink" xlink "arcrole")), + local_name!("xlink:href") => Some(qualname!("xlink" xlink "href")), + local_name!("xlink:role") => Some(qualname!("xlink" xlink "role")), + local_name!("xlink:show") => Some(qualname!("xlink" xlink "show")), + local_name!("xlink:title") => Some(qualname!("xlink" xlink "title")), + local_name!("xlink:type") => Some(qualname!("xlink" xlink "type")), + local_name!("xml:base") => Some(qualname!("xml" xml "base")), + local_name!("xml:lang") => Some(qualname!("xml" xml "lang")), + local_name!("xml:space") => Some(qualname!("xml" xml "space")), + local_name!("xmlns") => Some(qualname!("" xmlns "xmlns")), + local_name!("xmlns:xlink") => Some(qualname!("xmlns" xmlns "xlink")), + _ => None, + }); + } + + fn foreign_start_tag(&mut self, mut tag: Tag) -> ProcessResult<Handle> { + let current_ns = self.sink.elem_name(self.adjusted_current_node()).ns.clone(); + match current_ns { + ns!(mathml) => self.adjust_mathml_attributes(&mut tag), + ns!(svg) => { + self.adjust_svg_tag_name(&mut tag); + self.adjust_svg_attributes(&mut tag); + }, + _ => (), + } + self.adjust_foreign_attributes(&mut tag); + if tag.self_closing { + // FIXME(#118): <script /> in SVG + self.insert_element(NoPush, current_ns, tag.name, tag.attrs); + DoneAckSelfClosing + } else { + self.insert_element(Push, current_ns, tag.name, tag.attrs); + Done + } + } + + fn unexpected_start_tag_in_foreign_content(&mut self, tag: Tag) -> ProcessResult<Handle> { + self.unexpected(&tag); + if self.is_fragment() { + self.foreign_start_tag(tag) + } else { + self.pop(); + while !self.current_node_in(|n| { + *n.ns == ns!(html) || + mathml_text_integration_point(n) || + svg_html_integration_point(n) + }) { + self.pop(); + } + ReprocessForeign(TagToken(tag)) + } + } +} diff --git a/src/tree_builder/rules.rs b/src/tree_builder/rules.rs new file mode 100644 index 0000000..bdc8afd --- /dev/null +++ b/src/tree_builder/rules.rs @@ -0,0 +1,1449 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// The tree builder rules, as a single, enormous nested match expression. + +use markup5ever::{expanded_name, local_name, namespace_prefix, namespace_url, ns}; +use crate::tokenizer::states::{Plaintext, Rawtext, Rcdata, ScriptData}; +use crate::tree_builder::tag_sets::*; +use crate::tree_builder::types::*; + +use std::borrow::ToOwned; + +use crate::tendril::SliceExt; + +fn any_not_whitespace(x: &StrTendril) -> bool { + // FIXME: this might be much faster as a byte scan + x.chars().any(|c| !is_ascii_whitespace(c)) +} + +fn current_node<Handle>(open_elems: &[Handle]) -> &Handle { + open_elems.last().expect("no current element") +} + +#[doc(hidden)] +impl<Handle, Sink> TreeBuilder<Handle, Sink> +where + Handle: Clone, + Sink: TreeSink<Handle = Handle>, +{ + fn step(&mut self, mode: InsertionMode, token: Token) -> ProcessResult<Handle> { + self.debug_step(mode, &token); + + match mode { + //§ the-initial-insertion-mode + Initial => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, _) => Done, + CommentToken(text) => self.append_comment_to_doc(text), + token => { + if !self.opts.iframe_srcdoc { + self.unexpected(&token); + self.set_quirks_mode(Quirks); + } + Reprocess(BeforeHtml, token) + } + }), + + //§ the-before-html-insertion-mode + BeforeHtml => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, _) => Done, + CommentToken(text) => self.append_comment_to_doc(text), + + tag @ <html> => { + self.create_root(tag.attrs); + self.mode = BeforeHead; + Done + } + + </head> </body> </html> </br> => else, + + tag @ </_> => self.unexpected(&tag), + + token => { + self.create_root(vec!()); + Reprocess(BeforeHead, token) + } + }), + + //§ the-before-head-insertion-mode + BeforeHead => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, _) => Done, + CommentToken(text) => self.append_comment(text), + + <html> => self.step(InBody, token), + + tag @ <head> => { + self.head_elem = Some(self.insert_element_for(tag)); + self.mode = InHead; + Done + } + + </head> </body> </html> </br> => else, + + tag @ </_> => self.unexpected(&tag), + + token => { + self.head_elem = Some(self.insert_phantom(local_name!("head"))); + Reprocess(InHead, token) + } + }), + + //§ parsing-main-inhead + InHead => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, text) => self.append_text(text), + CommentToken(text) => self.append_comment(text), + + <html> => self.step(InBody, token), + + tag @ <base> <basefont> <bgsound> <link> <meta> => { + // FIXME: handle <meta charset=...> and <meta http-equiv="Content-Type"> + self.insert_and_pop_element_for(tag); + DoneAckSelfClosing + } + + tag @ <title> => { + self.parse_raw_data(tag, Rcdata) + } + + tag @ <noframes> <style> <noscript> => { + if (!self.opts.scripting_enabled) && (tag.name == local_name!("noscript")) { + self.insert_element_for(tag); + self.mode = InHeadNoscript; + Done + } else { + self.parse_raw_data(tag, Rawtext) + } + } + + tag @ <script> => { + let elem = create_element( + &mut self.sink, QualName::new(None, ns!(html), local_name!("script")), + tag.attrs); + if self.is_fragment() { + self.sink.mark_script_already_started(&elem); + } + self.insert_appropriately(AppendNode(elem.clone()), None); + self.open_elems.push(elem); + self.to_raw_text_mode(ScriptData) + } + + </head> => { + self.pop(); + self.mode = AfterHead; + Done + } + + </body> </html> </br> => else, + + tag @ <template> => { + self.insert_element_for(tag); + self.active_formatting.push(Marker); + self.frameset_ok = false; + self.mode = InTemplate; + self.template_modes.push(InTemplate); + Done + } + + tag @ </template> => { + if !self.in_html_elem_named(local_name!("template")) { + self.unexpected(&tag); + } else { + self.generate_implied_end(thorough_implied_end); + self.expect_to_close(local_name!("template")); + self.clear_active_formatting_to_marker(); + self.template_modes.pop(); + self.mode = self.reset_insertion_mode(); + } + Done + } + + <head> => self.unexpected(&token), + tag @ </_> => self.unexpected(&tag), + + token => { + self.pop(); + Reprocess(AfterHead, token) + } + }), + + //§ parsing-main-inheadnoscript + InHeadNoscript => match_token!(token { + <html> => self.step(InBody, token), + + </noscript> => { + self.pop(); + self.mode = InHead; + Done + }, + + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, _) => self.step(InHead, token), + + CommentToken(_) => self.step(InHead, token), + + <basefont> <bgsound> <link> <meta> <noframes> <style> + => self.step(InHead, token), + + </br> => else, + + <head> <noscript> => self.unexpected(&token), + tag @ </_> => self.unexpected(&tag), + + token => { + self.unexpected(&token); + self.pop(); + Reprocess(InHead, token) + }, + }), + + //§ the-after-head-insertion-mode + AfterHead => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, text) => self.append_text(text), + CommentToken(text) => self.append_comment(text), + + <html> => self.step(InBody, token), + + tag @ <body> => { + self.insert_element_for(tag); + self.frameset_ok = false; + self.mode = InBody; + Done + } + + tag @ <frameset> => { + self.insert_element_for(tag); + self.mode = InFrameset; + Done + } + + <base> <basefont> <bgsound> <link> <meta> + <noframes> <script> <style> <template> <title> => { + self.unexpected(&token); + let head = self.head_elem.as_ref().expect("no head element").clone(); + self.push(&head); + let result = self.step(InHead, token); + self.remove_from_stack(&head); + result + } + + </template> => self.step(InHead, token), + + </body> </html> </br> => else, + + <head> => self.unexpected(&token), + tag @ </_> => self.unexpected(&tag), + + token => { + self.insert_phantom(local_name!("body")); + Reprocess(InBody, token) + } + }), + + //§ parsing-main-inbody + InBody => match_token!(token { + NullCharacterToken => self.unexpected(&token), + + CharacterTokens(_, text) => { + self.reconstruct_formatting(); + if any_not_whitespace(&text) { + self.frameset_ok = false; + } + self.append_text(text) + } + + CommentToken(text) => self.append_comment(text), + + tag @ <html> => { + self.unexpected(&tag); + if !self.in_html_elem_named(local_name!("template")) { + let top = html_elem(&self.open_elems); + self.sink.add_attrs_if_missing(top, tag.attrs); + } + Done + } + + <base> <basefont> <bgsound> <link> <meta> <noframes> + <script> <style> <template> <title> </template> => { + self.step(InHead, token) + } + + tag @ <body> => { + self.unexpected(&tag); + match self.body_elem().cloned() { + Some(ref node) if self.open_elems.len() != 1 && + !self.in_html_elem_named(local_name!("template")) => { + self.frameset_ok = false; + self.sink.add_attrs_if_missing(node, tag.attrs) + }, + _ => {} + } + Done + } + + tag @ <frameset> => { + self.unexpected(&tag); + if !self.frameset_ok { return Done; } + + let body = unwrap_or_return!(self.body_elem(), Done).clone(); + self.sink.remove_from_parent(&body); + + // FIXME: can we get here in the fragment case? + // What to do with the first element then? + self.open_elems.truncate(1); + self.insert_element_for(tag); + self.mode = InFrameset; + Done + } + + EOFToken => { + if !self.template_modes.is_empty() { + self.step(InTemplate, token) + } else { + self.check_body_end(); + self.stop_parsing() + } + } + + </body> => { + if self.in_scope_named(default_scope, local_name!("body")) { + self.check_body_end(); + self.mode = AfterBody; + } else { + self.sink.parse_error(Borrowed("</body> with no <body> in scope")); + } + Done + } + + </html> => { + if self.in_scope_named(default_scope, local_name!("body")) { + self.check_body_end(); + Reprocess(AfterBody, token) + } else { + self.sink.parse_error(Borrowed("</html> with no <body> in scope")); + Done + } + } + + tag @ <address> <article> <aside> <blockquote> <center> <details> <dialog> + <dir> <div> <dl> <fieldset> <figcaption> <figure> <footer> <header> + <hgroup> <main> <nav> <ol> <p> <section> <summary> <ul> => { + self.close_p_element_in_button_scope(); + self.insert_element_for(tag); + Done + } + + tag @ <menu> => { + self.close_p_element_in_button_scope(); + self.insert_element_for(tag); + Done + } + + tag @ <h1> <h2> <h3> <h4> <h5> <h6> => { + self.close_p_element_in_button_scope(); + if self.current_node_in(heading_tag) { + self.sink.parse_error(Borrowed("nested heading tags")); + self.pop(); + } + self.insert_element_for(tag); + Done + } + + tag @ <pre> <listing> => { + self.close_p_element_in_button_scope(); + self.insert_element_for(tag); + self.ignore_lf = true; + self.frameset_ok = false; + Done + } + + tag @ <form> => { + if self.form_elem.is_some() && + !self.in_html_elem_named(local_name!("template")) { + self.sink.parse_error(Borrowed("nested forms")); + } else { + self.close_p_element_in_button_scope(); + let elem = self.insert_element_for(tag); + if !self.in_html_elem_named(local_name!("template")) { + self.form_elem = Some(elem); + } + } + Done + } + + tag @ <li> <dd> <dt> => { + declare_tag_set!(close_list = "li"); + declare_tag_set!(close_defn = "dd" "dt"); + declare_tag_set!(extra_special = [special_tag] - "address" "div" "p"); + let list = match tag.name { + local_name!("li") => true, + local_name!("dd") | local_name!("dt") => false, + _ => unreachable!(), + }; + + self.frameset_ok = false; + + let mut to_close = None; + for node in self.open_elems.iter().rev() { + let name = self.sink.elem_name(node); + let can_close = if list { + close_list(name) + } else { + close_defn(name) + }; + if can_close { + to_close = Some(name.local.clone()); + break; + } + if extra_special(name) { + break; + } + } + + match to_close { + Some(name) => { + self.generate_implied_end_except(name.clone()); + self.expect_to_close(name); + } + None => (), + } + + self.close_p_element_in_button_scope(); + self.insert_element_for(tag); + Done + } + + tag @ <plaintext> => { + self.close_p_element_in_button_scope(); + self.insert_element_for(tag); + ToPlaintext + } + + tag @ <button> => { + if self.in_scope_named(default_scope, local_name!("button")) { + self.sink.parse_error(Borrowed("nested buttons")); + self.generate_implied_end(cursory_implied_end); + self.pop_until_named(local_name!("button")); + } + self.reconstruct_formatting(); + self.insert_element_for(tag); + self.frameset_ok = false; + Done + } + + tag @ </address> </article> </aside> </blockquote> </button> </center> + </details> </dialog> </dir> </div> </dl> </fieldset> </figcaption> + </figure> </footer> </header> </hgroup> </listing> </main> </menu> + </nav> </ol> </pre> </section> </summary> </ul> => { + if !self.in_scope_named(default_scope, tag.name.clone()) { + self.unexpected(&tag); + } else { + self.generate_implied_end(cursory_implied_end); + self.expect_to_close(tag.name); + } + Done + } + + </form> => { + if !self.in_html_elem_named(local_name!("template")) { + // Can't use unwrap_or_return!() due to rust-lang/rust#16617. + let node = match self.form_elem.take() { + None => { + self.sink.parse_error(Borrowed("Null form element pointer on </form>")); + return Done; + } + Some(x) => x, + }; + if !self.in_scope(default_scope, |n| self.sink.same_node(&node, &n)) { + self.sink.parse_error(Borrowed("Form element not in scope on </form>")); + return Done; + } + self.generate_implied_end(cursory_implied_end); + let current = self.current_node().clone(); + self.remove_from_stack(&node); + if !self.sink.same_node(¤t, &node) { + self.sink.parse_error(Borrowed("Bad open element on </form>")); + } + } else { + if !self.in_scope_named(default_scope, local_name!("form")) { + self.sink.parse_error(Borrowed("Form element not in scope on </form>")); + return Done; + } + self.generate_implied_end(cursory_implied_end); + if !self.current_node_named(local_name!("form")) { + self.sink.parse_error(Borrowed("Bad open element on </form>")); + } + self.pop_until_named(local_name!("form")); + } + Done + } + + </p> => { + if !self.in_scope_named(button_scope, local_name!("p")) { + self.sink.parse_error(Borrowed("No <p> tag to close")); + self.insert_phantom(local_name!("p")); + } + self.close_p_element(); + Done + } + + tag @ </li> </dd> </dt> => { + let in_scope = if tag.name == local_name!("li") { + self.in_scope_named(list_item_scope, tag.name.clone()) + } else { + self.in_scope_named(default_scope, tag.name.clone()) + }; + if in_scope { + self.generate_implied_end_except(tag.name.clone()); + self.expect_to_close(tag.name); + } else { + self.sink.parse_error(Borrowed("No matching tag to close")); + } + Done + } + + tag @ </h1> </h2> </h3> </h4> </h5> </h6> => { + if self.in_scope(default_scope, |n| self.elem_in(&n, heading_tag)) { + self.generate_implied_end(cursory_implied_end); + if !self.current_node_named(tag.name) { + self.sink.parse_error(Borrowed("Closing wrong heading tag")); + } + self.pop_until(heading_tag); + } else { + self.sink.parse_error(Borrowed("No heading tag to close")); + } + Done + } + + tag @ <a> => { + self.handle_misnested_a_tags(&tag); + self.reconstruct_formatting(); + self.create_formatting_element_for(tag); + Done + } + + tag @ <b> <big> <code> <em> <font> <i> <s> <small> <strike> <strong> <tt> <u> => { + self.reconstruct_formatting(); + self.create_formatting_element_for(tag); + Done + } + + tag @ <nobr> => { + self.reconstruct_formatting(); + if self.in_scope_named(default_scope, local_name!("nobr")) { + self.sink.parse_error(Borrowed("Nested <nobr>")); + self.adoption_agency(local_name!("nobr")); + self.reconstruct_formatting(); + } + self.create_formatting_element_for(tag); + Done + } + + tag @ </a> </b> </big> </code> </em> </font> </i> </nobr> + </s> </small> </strike> </strong> </tt> </u> => { + self.adoption_agency(tag.name); + Done + } + + tag @ <applet> <marquee> <object> => { + self.reconstruct_formatting(); + self.insert_element_for(tag); + self.active_formatting.push(Marker); + self.frameset_ok = false; + Done + } + + tag @ </applet> </marquee> </object> => { + if !self.in_scope_named(default_scope, tag.name.clone()) { + self.unexpected(&tag); + } else { + self.generate_implied_end(cursory_implied_end); + self.expect_to_close(tag.name); + self.clear_active_formatting_to_marker(); + } + Done + } + + tag @ <table> => { + if self.quirks_mode != Quirks { + self.close_p_element_in_button_scope(); + } + self.insert_element_for(tag); + self.frameset_ok = false; + self.mode = InTable; + Done + } + + tag @ </br> => { + self.unexpected(&tag); + self.step(InBody, TagToken(Tag { + kind: StartTag, + attrs: vec!(), + ..tag + })) + } + + tag @ <area> <br> <embed> <img> <keygen> <wbr> <input> => { + let keep_frameset_ok = match tag.name { + local_name!("input") => self.is_type_hidden(&tag), + _ => false, + }; + self.reconstruct_formatting(); + self.insert_and_pop_element_for(tag); + if !keep_frameset_ok { + self.frameset_ok = false; + } + DoneAckSelfClosing + } + + tag @ <param> <source> <track> => { + self.insert_and_pop_element_for(tag); + DoneAckSelfClosing + } + + tag @ <hr> => { + self.close_p_element_in_button_scope(); + self.insert_and_pop_element_for(tag); + self.frameset_ok = false; + DoneAckSelfClosing + } + + tag @ <image> => { + self.unexpected(&tag); + self.step(InBody, TagToken(Tag { + name: local_name!("img"), + ..tag + })) + } + + tag @ <textarea> => { + self.ignore_lf = true; + self.frameset_ok = false; + self.parse_raw_data(tag, Rcdata) + } + + tag @ <xmp> => { + self.close_p_element_in_button_scope(); + self.reconstruct_formatting(); + self.frameset_ok = false; + self.parse_raw_data(tag, Rawtext) + } + + tag @ <iframe> => { + self.frameset_ok = false; + self.parse_raw_data(tag, Rawtext) + } + + tag @ <noembed> => { + self.parse_raw_data(tag, Rawtext) + } + + // <noscript> handled in wildcard case below + + tag @ <select> => { + self.reconstruct_formatting(); + self.insert_element_for(tag); + self.frameset_ok = false; + // NB: mode == InBody but possibly self.mode != mode, if + // we're processing "as in the rules for InBody". + self.mode = match self.mode { + InTable | InCaption | InTableBody + | InRow | InCell => InSelectInTable, + _ => InSelect, + }; + Done + } + + tag @ <optgroup> <option> => { + if self.current_node_named(local_name!("option")) { + self.pop(); + } + self.reconstruct_formatting(); + self.insert_element_for(tag); + Done + } + + tag @ <rb> <rtc> => { + if self.in_scope_named(default_scope, local_name!("ruby")) { + self.generate_implied_end(cursory_implied_end); + } + if !self.current_node_named(local_name!("ruby")) { + self.unexpected(&tag); + } + self.insert_element_for(tag); + Done + } + + tag @ <rp> <rt> => { + if self.in_scope_named(default_scope, local_name!("ruby")) { + self.generate_implied_end_except(local_name!("rtc")); + } + if !self.current_node_named(local_name!("rtc")) && !self.current_node_named(local_name!("ruby")) { + self.unexpected(&tag); + } + self.insert_element_for(tag); + Done + } + + tag @ <math> => self.enter_foreign(tag, ns!(mathml)), + + tag @ <svg> => self.enter_foreign(tag, ns!(svg)), + + <caption> <col> <colgroup> <frame> <head> + <tbody> <td> <tfoot> <th> <thead> <tr> => { + self.unexpected(&token); + Done + } + + tag @ <_> => { + if self.opts.scripting_enabled && tag.name == local_name!("noscript") { + self.parse_raw_data(tag, Rawtext) + } else { + self.reconstruct_formatting(); + self.insert_element_for(tag); + Done + } + } + + tag @ </_> => { + self.process_end_tag_in_body(tag); + Done + } + + // FIXME: This should be unreachable, but match_token requires a + // catch-all case. + _ => panic!("impossible case in InBody mode"), + }), + + //§ parsing-main-incdata + Text => match_token!(token { + CharacterTokens(_, text) => self.append_text(text), + + EOFToken => { + self.unexpected(&token); + if self.current_node_named(local_name!("script")) { + let current = current_node(&self.open_elems); + self.sink.mark_script_already_started(current); + } + self.pop(); + Reprocess(self.orig_mode.take().unwrap(), token) + } + + tag @ </_> => { + let node = self.pop(); + self.mode = self.orig_mode.take().unwrap(); + if tag.name == local_name!("script") { + return Script(node); + } + Done + } + + // The spec doesn't say what to do here. + // Other tokens are impossible? + _ => panic!("impossible case in Text mode"), + }), + + //§ parsing-main-intable + InTable => match_token!(token { + // FIXME: hack, should implement pat | pat for match_token instead + NullCharacterToken => self.process_chars_in_table(token), + + CharacterTokens(..) => self.process_chars_in_table(token), + + CommentToken(text) => self.append_comment(text), + + tag @ <caption> => { + self.pop_until_current(table_scope); + self.active_formatting.push(Marker); + self.insert_element_for(tag); + self.mode = InCaption; + Done + } + + tag @ <colgroup> => { + self.pop_until_current(table_scope); + self.insert_element_for(tag); + self.mode = InColumnGroup; + Done + } + + <col> => { + self.pop_until_current(table_scope); + self.insert_phantom(local_name!("colgroup")); + Reprocess(InColumnGroup, token) + } + + tag @ <tbody> <tfoot> <thead> => { + self.pop_until_current(table_scope); + self.insert_element_for(tag); + self.mode = InTableBody; + Done + } + + <td> <th> <tr> => { + self.pop_until_current(table_scope); + self.insert_phantom(local_name!("tbody")); + Reprocess(InTableBody, token) + } + + <table> => { + self.unexpected(&token); + if self.in_scope_named(table_scope, local_name!("table")) { + self.pop_until_named(local_name!("table")); + Reprocess(self.reset_insertion_mode(), token) + } else { + Done + } + } + + </table> => { + if self.in_scope_named(table_scope, local_name!("table")) { + self.pop_until_named(local_name!("table")); + self.mode = self.reset_insertion_mode(); + } else { + self.unexpected(&token); + } + Done + } + + </body> </caption> </col> </colgroup> </html> + </tbody> </td> </tfoot> </th> </thead> </tr> => + self.unexpected(&token), + + <style> <script> <template> </template> + => self.step(InHead, token), + + tag @ <input> => { + self.unexpected(&tag); + if self.is_type_hidden(&tag) { + self.insert_and_pop_element_for(tag); + DoneAckSelfClosing + } else { + self.foster_parent_in_body(TagToken(tag)) + } + } + + tag @ <form> => { + self.unexpected(&tag); + if !self.in_html_elem_named(local_name!("template")) && self.form_elem.is_none() { + self.form_elem = Some(self.insert_and_pop_element_for(tag)); + } + Done + } + + EOFToken => self.step(InBody, token), + + token => { + self.unexpected(&token); + self.foster_parent_in_body(token) + } + }), + + //§ parsing-main-intabletext + InTableText => match_token!(token { + NullCharacterToken => self.unexpected(&token), + + CharacterTokens(split, text) => { + self.pending_table_text.push((split, text)); + Done + } + + token => { + let pending = replace(&mut self.pending_table_text, vec!()); + let contains_nonspace = pending.iter().any(|&(split, ref text)| { + match split { + Whitespace => false, + NotWhitespace => true, + NotSplit => any_not_whitespace(text), + } + }); + + if contains_nonspace { + self.sink.parse_error(Borrowed("Non-space table text")); + for (split, text) in pending.into_iter() { + match self.foster_parent_in_body(CharacterTokens(split, text)) { + Done => (), + _ => panic!("not prepared to handle this!"), + } + } + } else { + for (_, text) in pending.into_iter() { + self.append_text(text); + } + } + + Reprocess(self.orig_mode.take().unwrap(), token) + } + }), + + //§ parsing-main-incaption + InCaption => match_token!(token { + tag @ <caption> <col> <colgroup> <tbody> <td> <tfoot> + <th> <thead> <tr> </table> </caption> => { + if self.in_scope_named(table_scope, local_name!("caption")) { + self.generate_implied_end(cursory_implied_end); + self.expect_to_close(local_name!("caption")); + self.clear_active_formatting_to_marker(); + match tag { + Tag { kind: EndTag, name: local_name!("caption"), .. } => { + self.mode = InTable; + Done + } + _ => Reprocess(InTable, TagToken(tag)) + } + } else { + self.unexpected(&tag); + Done + } + } + + </body> </col> </colgroup> </html> </tbody> + </td> </tfoot> </th> </thead> </tr> => self.unexpected(&token), + + token => self.step(InBody, token), + }), + + //§ parsing-main-incolgroup + InColumnGroup => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, text) => self.append_text(text), + CommentToken(text) => self.append_comment(text), + + <html> => self.step(InBody, token), + + tag @ <col> => { + self.insert_and_pop_element_for(tag); + DoneAckSelfClosing + } + + </colgroup> => { + if self.current_node_named(local_name!("colgroup")) { + self.pop(); + self.mode = InTable; + } else { + self.unexpected(&token); + } + Done + } + + </col> => self.unexpected(&token), + + <template> </template> => self.step(InHead, token), + + EOFToken => self.step(InBody, token), + + token => { + if self.current_node_named(local_name!("colgroup")) { + self.pop(); + Reprocess(InTable, token) + } else { + self.unexpected(&token) + } + } + }), + + //§ parsing-main-intbody + InTableBody => match_token!(token { + tag @ <tr> => { + self.pop_until_current(table_body_context); + self.insert_element_for(tag); + self.mode = InRow; + Done + } + + <th> <td> => { + self.unexpected(&token); + self.pop_until_current(table_body_context); + self.insert_phantom(local_name!("tr")); + Reprocess(InRow, token) + } + + tag @ </tbody> </tfoot> </thead> => { + if self.in_scope_named(table_scope, tag.name.clone()) { + self.pop_until_current(table_body_context); + self.pop(); + self.mode = InTable; + } else { + self.unexpected(&tag); + } + Done + } + + <caption> <col> <colgroup> <tbody> <tfoot> <thead> </table> => { + declare_tag_set!(table_outer = "table" "tbody" "tfoot"); + if self.in_scope(table_scope, |e| self.elem_in(&e, table_outer)) { + self.pop_until_current(table_body_context); + self.pop(); + Reprocess(InTable, token) + } else { + self.unexpected(&token) + } + } + + </body> </caption> </col> </colgroup> </html> </td> </th> </tr> + => self.unexpected(&token), + + token => self.step(InTable, token), + }), + + //§ parsing-main-intr + InRow => match_token!(token { + tag @ <th> <td> => { + self.pop_until_current(table_row_context); + self.insert_element_for(tag); + self.mode = InCell; + self.active_formatting.push(Marker); + Done + } + + </tr> => { + if self.in_scope_named(table_scope, local_name!("tr")) { + self.pop_until_current(table_row_context); + let node = self.pop(); + self.assert_named(&node, local_name!("tr")); + self.mode = InTableBody; + } else { + self.unexpected(&token); + } + Done + } + + <caption> <col> <colgroup> <tbody> <tfoot> <thead> <tr> </table> => { + if self.in_scope_named(table_scope, local_name!("tr")) { + self.pop_until_current(table_row_context); + let node = self.pop(); + self.assert_named(&node, local_name!("tr")); + Reprocess(InTableBody, token) + } else { + self.unexpected(&token) + } + } + + tag @ </tbody> </tfoot> </thead> => { + if self.in_scope_named(table_scope, tag.name.clone()) { + if self.in_scope_named(table_scope, local_name!("tr")) { + self.pop_until_current(table_row_context); + let node = self.pop(); + self.assert_named(&node, local_name!("tr")); + Reprocess(InTableBody, TagToken(tag)) + } else { + Done + } + } else { + self.unexpected(&tag) + } + } + + </body> </caption> </col> </colgroup> </html> </td> </th> + => self.unexpected(&token), + + token => self.step(InTable, token), + }), + + //§ parsing-main-intd + InCell => match_token!(token { + tag @ </td> </th> => { + if self.in_scope_named(table_scope, tag.name.clone()) { + self.generate_implied_end(cursory_implied_end); + self.expect_to_close(tag.name); + self.clear_active_formatting_to_marker(); + self.mode = InRow; + } else { + self.unexpected(&tag); + } + Done + } + + <caption> <col> <colgroup> <tbody> <td> <tfoot> <th> <thead> <tr> => { + if self.in_scope(table_scope, |n| self.elem_in(&n, td_th)) { + self.close_the_cell(); + Reprocess(InRow, token) + } else { + self.unexpected(&token) + } + } + + </body> </caption> </col> </colgroup> </html> + => self.unexpected(&token), + + tag @ </table> </tbody> </tfoot> </thead> </tr> => { + if self.in_scope_named(table_scope, tag.name.clone()) { + self.close_the_cell(); + Reprocess(InRow, TagToken(tag)) + } else { + self.unexpected(&tag) + } + } + + token => self.step(InBody, token), + }), + + //§ parsing-main-inselect + InSelect => match_token!(token { + NullCharacterToken => self.unexpected(&token), + CharacterTokens(_, text) => self.append_text(text), + CommentToken(text) => self.append_comment(text), + + <html> => self.step(InBody, token), + + tag @ <option> => { + if self.current_node_named(local_name!("option")) { + self.pop(); + } + self.insert_element_for(tag); + Done + } + + tag @ <optgroup> => { + if self.current_node_named(local_name!("option")) { + self.pop(); + } + if self.current_node_named(local_name!("optgroup")) { + self.pop(); + } + self.insert_element_for(tag); + Done + } + + </optgroup> => { + if self.open_elems.len() >= 2 + && self.current_node_named(local_name!("option")) + && self.html_elem_named(&self.open_elems[self.open_elems.len() - 2], + local_name!("optgroup")) { + self.pop(); + } + if self.current_node_named(local_name!("optgroup")) { + self.pop(); + } else { + self.unexpected(&token); + } + Done + } + + </option> => { + if self.current_node_named(local_name!("option")) { + self.pop(); + } else { + self.unexpected(&token); + } + Done + } + + tag @ <select> </select> => { + let in_scope = self.in_scope_named(select_scope, local_name!("select")); + + if !in_scope || tag.kind == StartTag { + self.unexpected(&tag); + } + + if in_scope { + self.pop_until_named(local_name!("select")); + self.mode = self.reset_insertion_mode(); + } + Done + } + + <input> <keygen> <textarea> => { + self.unexpected(&token); + if self.in_scope_named(select_scope, local_name!("select")) { + self.pop_until_named(local_name!("select")); + Reprocess(self.reset_insertion_mode(), token) + } else { + Done + } + } + + <script> <template> </template> => self.step(InHead, token), + + EOFToken => self.step(InBody, token), + + token => self.unexpected(&token), + }), + + //§ parsing-main-inselectintable + InSelectInTable => match_token!(token { + <caption> <table> <tbody> <tfoot> <thead> <tr> <td> <th> => { + self.unexpected(&token); + self.pop_until_named(local_name!("select")); + Reprocess(self.reset_insertion_mode(), token) + } + + tag @ </caption> </table> </tbody> </tfoot> </thead> </tr> </td> </th> => { + self.unexpected(&tag); + if self.in_scope_named(table_scope, tag.name.clone()) { + self.pop_until_named(local_name!("select")); + Reprocess(self.reset_insertion_mode(), TagToken(tag)) + } else { + Done + } + } + + token => self.step(InSelect, token), + }), + + //§ parsing-main-intemplate + InTemplate => match_token!(token { + CharacterTokens(_, _) => self.step(InBody, token), + CommentToken(_) => self.step(InBody, token), + + <base> <basefont> <bgsound> <link> <meta> <noframes> <script> + <style> <template> <title> </template> => { + self.step(InHead, token) + } + + <caption> <colgroup> <tbody> <tfoot> <thead> => { + self.template_modes.pop(); + self.template_modes.push(InTable); + Reprocess(InTable, token) + } + + <col> => { + self.template_modes.pop(); + self.template_modes.push(InColumnGroup); + Reprocess(InColumnGroup, token) + } + + <tr> => { + self.template_modes.pop(); + self.template_modes.push(InTableBody); + Reprocess(InTableBody, token) + } + + <td> <th> => { + self.template_modes.pop(); + self.template_modes.push(InRow); + Reprocess(InRow, token) + } + + EOFToken => { + if !self.in_html_elem_named(local_name!("template")) { + self.stop_parsing() + } else { + self.unexpected(&token); + self.pop_until_named(local_name!("template")); + self.clear_active_formatting_to_marker(); + self.template_modes.pop(); + self.mode = self.reset_insertion_mode(); + Reprocess(self.reset_insertion_mode(), token) + } + } + + tag @ <_> => { + self.template_modes.pop(); + self.template_modes.push(InBody); + Reprocess(InBody, TagToken(tag)) + } + + token => self.unexpected(&token), + }), + + //§ parsing-main-afterbody + AfterBody => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, _) => self.step(InBody, token), + CommentToken(text) => self.append_comment_to_html(text), + + <html> => self.step(InBody, token), + + </html> => { + if self.is_fragment() { + self.unexpected(&token); + } else { + self.mode = AfterAfterBody; + } + Done + } + + EOFToken => self.stop_parsing(), + + token => { + self.unexpected(&token); + Reprocess(InBody, token) + } + }), + + //§ parsing-main-inframeset + InFrameset => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, text) => self.append_text(text), + CommentToken(text) => self.append_comment(text), + + <html> => self.step(InBody, token), + + tag @ <frameset> => { + self.insert_element_for(tag); + Done + } + + </frameset> => { + if self.open_elems.len() == 1 { + self.unexpected(&token); + } else { + self.pop(); + if !self.is_fragment() && !self.current_node_named(local_name!("frameset")) { + self.mode = AfterFrameset; + } + } + Done + } + + tag @ <frame> => { + self.insert_and_pop_element_for(tag); + DoneAckSelfClosing + } + + <noframes> => self.step(InHead, token), + + EOFToken => { + if self.open_elems.len() != 1 { + self.unexpected(&token); + } + self.stop_parsing() + } + + token => self.unexpected(&token), + }), + + //§ parsing-main-afterframeset + AfterFrameset => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, text) => self.append_text(text), + CommentToken(text) => self.append_comment(text), + + <html> => self.step(InBody, token), + + </html> => { + self.mode = AfterAfterFrameset; + Done + } + + <noframes> => self.step(InHead, token), + + EOFToken => self.stop_parsing(), + + token => self.unexpected(&token), + }), + + //§ the-after-after-body-insertion-mode + AfterAfterBody => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, _) => self.step(InBody, token), + CommentToken(text) => self.append_comment_to_doc(text), + + <html> => self.step(InBody, token), + + EOFToken => self.stop_parsing(), + + token => { + self.unexpected(&token); + Reprocess(InBody, token) + } + }), + + //§ the-after-after-frameset-insertion-mode + AfterAfterFrameset => match_token!(token { + CharacterTokens(NotSplit, text) => SplitWhitespace(text), + CharacterTokens(Whitespace, _) => self.step(InBody, token), + CommentToken(text) => self.append_comment_to_doc(text), + + <html> => self.step(InBody, token), + + EOFToken => self.stop_parsing(), + + <noframes> => self.step(InHead, token), + + token => self.unexpected(&token), + }), + //§ END + } + } + + fn step_foreign(&mut self, token: Token) -> ProcessResult<Handle> { + match_token!(token { + NullCharacterToken => { + self.unexpected(&token); + self.append_text("\u{fffd}".to_tendril()) + } + + CharacterTokens(_, text) => { + if any_not_whitespace(&text) { + self.frameset_ok = false; + } + self.append_text(text) + } + + CommentToken(text) => self.append_comment(text), + + tag @ <b> <big> <blockquote> <body> <br> <center> <code> <dd> <div> <dl> + <dt> <em> <embed> <h1> <h2> <h3> <h4> <h5> <h6> <head> <hr> <i> + <img> <li> <listing> <menu> <meta> <nobr> <ol> <p> <pre> <ruby> + <s> <small> <span> <strong> <strike> <sub> <sup> <table> <tt> + <u> <ul> <var> => self.unexpected_start_tag_in_foreign_content(tag), + + tag @ <font> => { + let unexpected = tag.attrs.iter().any(|attr| { + matches!(attr.name.expanded(), + expanded_name!("", "color") | + expanded_name!("", "face") | + expanded_name!("", "size")) + }); + if unexpected { + self.unexpected_start_tag_in_foreign_content(tag) + } else { + self.foreign_start_tag(tag) + } + } + + tag @ <_> => self.foreign_start_tag(tag), + + // FIXME(#118): </script> in SVG + + tag @ </_> => { + let mut first = true; + let mut stack_idx = self.open_elems.len() - 1; + loop { + if stack_idx == 0 { + return Done; + } + + let html; + let eq; + { + let node_name = self.sink.elem_name(&self.open_elems[stack_idx]); + html = *node_name.ns == ns!(html); + eq = node_name.local.eq_ignore_ascii_case(&tag.name); + } + if !first && html { + let mode = self.mode; + return self.step(mode, TagToken(tag)); + } + + if eq { + self.open_elems.truncate(stack_idx); + return Done; + } + + if first { + self.unexpected(&tag); + first = false; + } + stack_idx -= 1; + } + } + + // FIXME: This should be unreachable, but match_token requires a + // catch-all case. + _ => panic!("impossible case in foreign content"), + }) + } +} diff --git a/src/tree_builder/tag_sets.rs b/src/tree_builder/tag_sets.rs new file mode 100644 index 0000000..377b34c --- /dev/null +++ b/src/tree_builder/tag_sets.rs @@ -0,0 +1,115 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Various sets of HTML tag names, and macros for declaring them. + +use crate::ExpandedName; +use mac::{_tt_as_expr_hack, matches}; +use markup5ever::{expanded_name, local_name, namespace_prefix, namespace_url, ns}; + +macro_rules! declare_tag_set_impl ( ($param:ident, $b:ident, $supr:ident, $($tag:tt)+) => ( + match $param { + $( expanded_name!(html $tag) => $b, )+ + p => $supr(p), + } +)); + +macro_rules! declare_tag_set_body ( + ($param:ident = [$supr:ident] - $($tag:tt)+) + => ( declare_tag_set_impl!($param, false, $supr, $($tag)+) ); + + ($param:ident = [$supr:ident] + $($tag:tt)+) + => ( declare_tag_set_impl!($param, true, $supr, $($tag)+) ); + + ($param:ident = $($tag:tt)+) + => ( declare_tag_set_impl!($param, true, empty_set, $($tag)+) ); +); + +macro_rules! declare_tag_set ( + (pub $name:ident = $($toks:tt)+) => ( + pub fn $name(p: crate::ExpandedName) -> bool { + declare_tag_set_body!(p = $($toks)+) + } + ); + + ($name:ident = $($toks:tt)+) => ( + fn $name(p: crate::ExpandedName) -> bool { + declare_tag_set_body!(p = $($toks)+) + } + ); +); + +#[inline(always)] +pub fn empty_set(_: ExpandedName) -> bool { + false +} +#[inline(always)] +pub fn full_set(_: ExpandedName) -> bool { + true +} + +declare_tag_set!(pub html_default_scope = + "applet" "caption" "html" "table" "td" "th" "marquee" "object" "template"); + +#[inline(always)] +pub fn default_scope(name: ExpandedName) -> bool { + html_default_scope(name) || + mathml_text_integration_point(name) || + svg_html_integration_point(name) +} + +declare_tag_set!(pub list_item_scope = [default_scope] + "ol" "ul"); +declare_tag_set!(pub button_scope = [default_scope] + "button"); +declare_tag_set!(pub table_scope = "html" "table" "template"); +declare_tag_set!(pub select_scope = [full_set] - "optgroup" "option"); + +declare_tag_set!(pub table_body_context = "tbody" "tfoot" "thead" "template" "html"); +declare_tag_set!(pub table_row_context = "tr" "template" "html"); +declare_tag_set!(pub td_th = "td" "th"); + +declare_tag_set!(pub cursory_implied_end = + "dd" "dt" "li" "option" "optgroup" "p" "rb" "rp" "rt" "rtc"); + +declare_tag_set!(pub thorough_implied_end = [cursory_implied_end] + + "caption" "colgroup" "tbody" "td" "tfoot" "th" "thead" "tr"); + +declare_tag_set!(pub heading_tag = "h1" "h2" "h3" "h4" "h5" "h6"); + +declare_tag_set!(pub special_tag = + "address" "applet" "area" "article" "aside" "base" "basefont" "bgsound" "blockquote" "body" + "br" "button" "caption" "center" "col" "colgroup" "dd" "details" "dir" "div" "dl" "dt" "embed" + "fieldset" "figcaption" "figure" "footer" "form" "frame" "frameset" "h1" "h2" "h3" "h4" "h5" + "h6" "head" "header" "hgroup" "hr" "html" "iframe" "img" "input" "isindex" "li" "link" + "listing" "main" "marquee" "menu" "meta" "nav" "noembed" "noframes" "noscript" + "object" "ol" "p" "param" "plaintext" "pre" "script" "section" "select" "source" "style" + "summary" "table" "tbody" "td" "template" "textarea" "tfoot" "th" "thead" "title" "tr" "track" + "ul" "wbr" "xmp"); +//§ END + +pub fn mathml_text_integration_point(p: ExpandedName) -> bool { + matches!( + p, + expanded_name!(mathml "mi") | + expanded_name!(mathml "mo") | + expanded_name!(mathml "mn") | + expanded_name!(mathml "ms") | + expanded_name!(mathml "mtext") + ) +} + +/// https://html.spec.whatwg.org/multipage/#html-integration-point +pub fn svg_html_integration_point(p: ExpandedName) -> bool { + // annotation-xml are handle in another place + matches!( + p, + expanded_name!(svg "foreignObject") | + expanded_name!(svg "desc") | + expanded_name!(svg "title") + ) +} diff --git a/src/tree_builder/types.rs b/src/tree_builder/types.rs new file mode 100644 index 0000000..e47d69b --- /dev/null +++ b/src/tree_builder/types.rs @@ -0,0 +1,95 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! Types used within the tree builder code. Not exported to users. + +use crate::tokenizer::states::RawKind; +use crate::tokenizer::Tag; + +use crate::tendril::StrTendril; + +pub use self::FormatEntry::*; +pub use self::InsertionMode::*; +pub use self::InsertionPoint::*; +pub use self::ProcessResult::*; +pub use self::SplitStatus::*; +pub use self::Token::*; + +#[derive(PartialEq, Eq, Copy, Clone, Debug)] +pub enum InsertionMode { + Initial, + BeforeHtml, + BeforeHead, + InHead, + InHeadNoscript, + AfterHead, + InBody, + Text, + InTable, + InTableText, + InCaption, + InColumnGroup, + InTableBody, + InRow, + InCell, + InSelect, + InSelectInTable, + InTemplate, + AfterBody, + InFrameset, + AfterFrameset, + AfterAfterBody, + AfterAfterFrameset, +} + +#[derive(PartialEq, Eq, Copy, Clone, Debug)] +pub enum SplitStatus { + NotSplit, + Whitespace, + NotWhitespace, +} + +/// A subset/refinement of `tokenizer::Token`. Everything else is handled +/// specially at the beginning of `process_token`. +#[derive(PartialEq, Eq, Clone, Debug)] +pub enum Token { + TagToken(Tag), + CommentToken(StrTendril), + CharacterTokens(SplitStatus, StrTendril), + NullCharacterToken, + EOFToken, +} + +pub enum ProcessResult<Handle> { + Done, + DoneAckSelfClosing, + SplitWhitespace(StrTendril), + Reprocess(InsertionMode, Token), + ReprocessForeign(Token), + Script(Handle), + ToPlaintext, + ToRawData(RawKind), +} + +pub enum FormatEntry<Handle> { + Element(Handle, Tag), + Marker, +} + +pub enum InsertionPoint<Handle> { + /// Insert as last child in this parent. + LastChild(Handle), + /// Insert before this following sibling. + BeforeSibling(Handle), + /// Insertion point is decided based on existence of element's parent node. + TableFosterParenting { + element: Handle, + prev_element: Handle, + }, +} diff --git a/src/util/str.rs b/src/util/str.rs new file mode 100644 index 0000000..b2eb41a --- /dev/null +++ b/src/util/str.rs @@ -0,0 +1,60 @@ +// Copyright 2014-2017 The html5ever Project Developers. See the +// COPYRIGHT file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use mac::{_tt_as_expr_hack, matches}; +use std::fmt; + +pub fn to_escaped_string<T: fmt::Debug>(x: &T) -> String { + // FIXME: don't allocate twice + let string = format!("{:?}", x); + string.chars().flat_map(|c| c.escape_default()).collect() +} + +/// If `c` is an ASCII letter, return the corresponding lowercase +/// letter, otherwise None. +pub fn lower_ascii_letter(c: char) -> Option<char> { + match c { + 'a'..='z' => Some(c), + 'A'..='Z' => Some((c as u8 - b'A' + b'a') as char), + _ => None, + } +} + +/// Is the character an ASCII alphanumeric character? +pub fn is_ascii_alnum(c: char) -> bool { + matches!(c, '0'..='9' | 'a'..='z' | 'A'..='Z') +} + +/// ASCII whitespace characters, as defined by +/// tree construction modes that treat them specially. +pub fn is_ascii_whitespace(c: char) -> bool { + matches!(c, '\t' | '\r' | '\n' | '\x0C' | ' ') +} + +#[cfg(test)] +#[allow(non_snake_case)] +mod test { + use super::{is_ascii_alnum, lower_ascii_letter}; + use mac::test_eq; + + test_eq!(lower_letter_a_is_a, lower_ascii_letter('a'), Some('a')); + test_eq!(lower_letter_A_is_a, lower_ascii_letter('A'), Some('a')); + test_eq!(lower_letter_symbol_is_None, lower_ascii_letter('!'), None); + test_eq!( + lower_letter_nonascii_is_None, + lower_ascii_letter('\u{a66e}'), + None + ); + + test_eq!(is_alnum_a, is_ascii_alnum('a'), true); + test_eq!(is_alnum_A, is_ascii_alnum('A'), true); + test_eq!(is_alnum_1, is_ascii_alnum('1'), true); + test_eq!(is_not_alnum_symbol, is_ascii_alnum('!'), false); + test_eq!(is_not_alnum_nonascii, is_ascii_alnum('\u{a66e}'), false); +} |