aboutsummaryrefslogtreecommitdiff
path: root/src/lib.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/lib.rs')
-rw-r--r--src/lib.rs2173
1 files changed, 4 insertions, 2169 deletions
diff --git a/src/lib.rs b/src/lib.rs
index 7722197..f3e0431 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -9,2179 +9,14 @@ mod error;
mod machine;
mod never;
mod reader;
+mod tokenizer;
+mod utils;
#[cfg(feature = "integration-tests")]
-pub use machine::State;
-#[cfg(not(feature = "integration-tests"))]
-use machine::State;
-
-use machine::{
- ascii_digit_pat, control_pat, noncharacter_pat, surrogate_pat, whitespace_pat, ControlToken,
-};
+pub use utils::State;
pub use emitter::{DefaultEmitter, Doctype, Emitter, EndTag, StartTag, Token};
pub use error::Error;
pub use never::Never;
pub use reader::{BufReadReader, Readable, Reader, StringReader};
-
-macro_rules! ctostr {
- ($c:expr) => {
- &*$c.encode_utf8(&mut [0; 4])
- };
-}
-
-// this is a stack that can hold 0 to 2 Ts
-#[derive(Debug, Default)]
-struct Stack2<T: Copy>(Option<(T, Option<T>)>);
-
-impl<T: Copy> Stack2<T> {
- #[inline]
- fn push(&mut self, c: T) {
- self.0 = match self.0 {
- None => Some((c, None)),
- Some((c1, None)) => Some((c1, Some(c))),
- Some((_c1, Some(_c2))) => panic!("stack full!"),
- }
- }
-
- #[inline]
- fn pop(&mut self) -> Option<T> {
- let (new_self, rv) = match self.0 {
- Some((c1, Some(c2))) => (Some((c1, None)), Some(c2)),
- Some((c1, None)) => (None, Some(c1)),
- None => (None, None),
- };
- self.0 = new_self;
- rv
- }
-
- #[inline]
- fn is_empty(&self) -> bool {
- matches!(self.0, None)
- }
-}
-
-/// A HTML tokenizer. See crate-level docs for basic usage.
-pub struct Tokenizer<R: Reader, E: Emitter = DefaultEmitter> {
- eof: bool,
- state: State,
- emitter: E,
- temporary_buffer: String,
- reader: R,
- to_reconsume: Stack2<Option<char>>,
- character_reference_code: u32,
- return_state: Option<State>,
-}
-
-impl<R: Reader> Tokenizer<R> {
- /// Create a new tokenizer from some input.
- ///
- /// `input` can be `&String` or `&str` at the moment, as those are the types for which
- /// [`crate::Readable`] is implemented, but you can implement that trait on your own types.
- ///
- /// Patches are welcome for providing an efficient implementation over async streams,
- /// iterators, files, etc, as long as any dependencies come behind featureflags.
- pub fn new<'a, S: Readable<'a, Reader = R>>(input: S) -> Self {
- Tokenizer::<S::Reader>::new_with_emitter(input, DefaultEmitter::default())
- }
-}
-
-impl<R: Reader, E: Emitter> Tokenizer<R, E> {
- /// Construct a new tokenizer from some input and a custom emitter.
- ///
- /// Use this method over [`Tokenizer::new`] when you want to have more control over string allocation for
- /// tokens.
- pub fn new_with_emitter<'a, S: Readable<'a, Reader = R>>(input: S, emitter: E) -> Self {
- Tokenizer {
- eof: false,
- state: State::Data,
- emitter,
- temporary_buffer: String::new(),
- to_reconsume: Stack2::default(),
- reader: input.to_reader(),
- character_reference_code: 0,
- return_state: None,
- }
- }
-
- #[cfg(feature = "integration-tests")]
- /// Test-internal function to override internal state.
- ///
- /// Only available with the `integration-tests` feature which is not public API.
- pub fn set_state(&mut self, state: State) {
- self.state = state;
- }
-
- /// Set the statemachine to start/continue in [plaintext
- /// state](https://html.spec.whatwg.org/#plaintext-state).
- ///
- /// This tokenizer never gets into that state naturally.
- pub fn set_plaintext_state(&mut self) {
- self.state = State::PlainText;
- }
-
- #[cfg(feature = "integration-tests")]
- /// Test-internal function to override internal state.
- ///
- /// Only available with the `integration-tests` feature which is not public API.
- pub fn set_last_start_tag(&mut self, last_start_tag: Option<&str>) {
- self.emitter.set_last_start_tag(last_start_tag);
- }
-
- #[inline]
- fn unread_char(&mut self, c: Option<char>) {
- self.to_reconsume.push(c);
- }
-
- #[inline]
- fn validate_char(&mut self, c: char) {
- match c as u32 {
- surrogate_pat!() => {
- self.emitter.emit_error(Error::SurrogateInInputStream);
- }
- noncharacter_pat!() => {
- self.emitter.emit_error(Error::NoncharacterInInputStream);
- }
- // control without whitespace or nul
- x @ control_pat!()
- if !matches!(x, 0x0000 | 0x0009 | 0x000a | 0x000c | 0x000d | 0x0020) =>
- {
- self.emitter
- .emit_error(Error::ControlCharacterInInputStream);
- }
- _ => (),
- }
- }
-
- fn read_char(&mut self) -> Result<Option<char>, R::Error> {
- let (c_res, reconsumed) = match self.to_reconsume.pop() {
- Some(c) => (Ok(c), true),
- None => (self.reader.read_char(), false),
- };
-
- let mut c = match c_res {
- Ok(Some(c)) => c,
- res => return res,
- };
-
- if c == '\r' {
- c = '\n';
- let c2 = self.reader.read_char()?;
- if c2 != Some('\n') {
- self.unread_char(c2);
- }
- }
-
- if !reconsumed {
- self.validate_char(c);
- }
-
- Ok(Some(c))
- }
-
- #[inline]
- fn try_read_string(&mut self, s: &str, case_sensitive: bool) -> Result<bool, R::Error> {
- debug_assert!(!s.is_empty());
- debug_assert!(self.to_reconsume.is_empty());
- self.reader.try_read_string(s, case_sensitive)
- }
-
- fn is_consumed_as_part_of_an_attribute(&self) -> bool {
- matches!(
- self.return_state,
- Some(
- State::AttributeValueDoubleQuoted
- | State::AttributeValueSingleQuoted
- | State::AttributeValueUnquoted
- )
- )
- }
-
- fn flush_code_points_consumed_as_character_reference(&mut self) {
- if self.is_consumed_as_part_of_an_attribute() {
- self.emitter.push_attribute_value(&self.temporary_buffer);
- self.temporary_buffer.clear();
- } else {
- self.flush_buffer_characters();
- }
- }
-
- fn next_input_character(&mut self) -> Result<Option<char>, R::Error> {
- let rv = self.read_char()?;
- self.unread_char(rv);
- Ok(rv)
- }
-
- fn flush_buffer_characters(&mut self) {
- self.emitter.emit_string(&self.temporary_buffer);
- self.temporary_buffer.clear();
- }
-
- fn consume(&mut self) -> Result<ControlToken, R::Error> {
- macro_rules! mutate_character_reference {
- (* $mul:literal + $x:ident - $sub:literal) => {
- match self
- .character_reference_code
- .checked_mul($mul)
- .and_then(|cr| cr.checked_add($x as u32 - $sub))
- {
- Some(cr) => self.character_reference_code = cr,
- None => {
- // provoke err
- self.character_reference_code = 0x110000;
- }
- };
- };
- }
-
- match self.state {
- State::Data => match self.read_char()? {
- Some('&') => {
- self.return_state = Some(self.state);
- self.state = State::CharacterReference;
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.state = State::TagOpen;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.emit_string("\0");
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- None => Ok(ControlToken::Eof),
- },
- State::RcData => match self.read_char()? {
- Some('&') => {
- self.return_state = Some(State::RcData);
- self.state = State::CharacterReference;
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.state = State::RcDataLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- None => Ok(ControlToken::Eof),
- },
- State::RawText => match self.read_char()? {
- Some('<') => {
- self.state = State::RawTextLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- None => Ok(ControlToken::Eof),
- },
- State::ScriptData => match self.read_char()? {
- Some('<') => {
- self.state = State::ScriptDataLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- None => Ok(ControlToken::Eof),
- },
- State::PlainText => match self.read_char()? {
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- None => Ok(ControlToken::Eof),
- },
- State::TagOpen => match self.read_char()? {
- Some('!') => {
- self.state = State::MarkupDeclarationOpen;
- Ok(ControlToken::Continue)
- }
- Some('/') => {
- self.state = State::EndTagOpen;
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.init_start_tag();
- self.state = State::TagName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- c @ Some('?') => {
- self.emitter
- .emit_error(Error::UnexpectedQuestionMarkInsteadOfTagName);
- self.emitter.init_comment();
- self.state = State::BogusComment;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofBeforeTagName);
- self.emitter.emit_string("<");
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::InvalidFirstCharacterOfTagName);
- self.state = State::Data;
- self.emitter.emit_string("<");
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::EndTagOpen => match self.read_char()? {
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.init_end_tag();
- self.state = State::TagName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter.emit_error(Error::MissingEndTagName);
- self.state = State::Data;
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofBeforeTagName);
- self.emitter.emit_string("</");
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter
- .emit_error(Error::InvalidFirstCharacterOfTagName);
- self.emitter.init_comment();
- self.state = State::BogusComment;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- },
- State::TagName => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::BeforeAttributeName;
- Ok(ControlToken::Continue)
- }
- Some('/') => {
- self.state = State::SelfClosingStartTag;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_tag_name("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.push_tag_name(ctostr!(x.to_ascii_lowercase()));
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInTag);
- Ok(ControlToken::Eof)
- }
- },
- State::RcDataLessThanSign => match self.read_char()? {
- Some('/') => {
- self.temporary_buffer.clear();
- self.state = State::RcDataEndTagOpen;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("<");
- self.state = State::RcData;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::RcDataEndTagOpen => match self.read_char()? {
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.init_end_tag();
- self.state = State::RcDataEndTagName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.state = State::RcData;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::RcDataEndTagName => match self.read_char()? {
- Some(whitespace_pat!()) if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::BeforeAttributeName;
- Ok(ControlToken::Continue)
- }
- Some('/') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::SelfClosingStartTag;
- Ok(ControlToken::Continue)
- }
- Some('>') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.push_tag_name(ctostr!(x.to_ascii_lowercase()));
- self.temporary_buffer.push(x);
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.flush_buffer_characters();
-
- self.state = State::RcData;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::RawTextLessThanSign => match self.read_char()? {
- Some('/') => {
- self.temporary_buffer.clear();
- self.state = State::RawTextEndTagOpen;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("<");
- self.state = State::RawText;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::RawTextEndTagOpen => match self.read_char()? {
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.init_end_tag();
- self.state = State::RawTextEndTagName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.state = State::RawText;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::RawTextEndTagName => match self.read_char()? {
- Some(whitespace_pat!()) if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::BeforeAttributeName;
- Ok(ControlToken::Continue)
- }
- Some('/') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::SelfClosingStartTag;
- Ok(ControlToken::Continue)
- }
- Some('>') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.push_tag_name(ctostr!(x.to_ascii_lowercase()));
- self.temporary_buffer.push(x);
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.flush_buffer_characters();
-
- self.state = State::RawText;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataLessThanSign => match self.read_char()? {
- Some('/') => {
- self.temporary_buffer.clear();
- self.state = State::ScriptDataEndTagOpen;
- Ok(ControlToken::Continue)
- }
- Some('!') => {
- self.state = State::ScriptDataEscapeStart;
- self.emitter.emit_string("<!");
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("<");
- self.state = State::Data;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEndTagOpen => match self.read_char()? {
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.init_end_tag();
- self.state = State::ScriptDataEndTagName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.state = State::ScriptData;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEndTagName => match self.read_char()? {
- Some(whitespace_pat!()) if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::BeforeAttributeName;
- Ok(ControlToken::Continue)
- }
- Some('/') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::SelfClosingStartTag;
- Ok(ControlToken::Continue)
- }
- Some('>') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.push_tag_name(ctostr!(x.to_ascii_lowercase()));
- self.temporary_buffer.push(x.to_ascii_lowercase());
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.flush_buffer_characters();
- self.state = State::Data;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscapeStart => match self.read_char()? {
- Some('-') => {
- self.state = State::ScriptDataEscapeStartDash;
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- c => {
- self.state = State::ScriptData;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscapeStartDash => match self.read_char()? {
- Some('-') => {
- self.state = State::ScriptDataEscapedDashDash;
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- c => {
- self.state = State::ScriptData;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscaped => match self.read_char()? {
- Some('-') => {
- self.state = State::ScriptDataEscapedDash;
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.state = State::ScriptDataEscapedLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter
- .emit_error(Error::EofInScriptHtmlCommentLikeText);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscapedDash => match self.read_char()? {
- Some('-') => {
- self.state = State::ScriptDataEscapedDashDash;
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.state = State::ScriptDataEscapedLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.state = State::ScriptDataEscaped;
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter
- .emit_error(Error::EofInScriptHtmlCommentLikeText);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.state = State::ScriptDataEscaped;
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscapedDashDash => match self.read_char()? {
- Some('-') => {
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.state = State::ScriptDataEscapedLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::ScriptData;
- self.emitter.emit_string(">");
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.state = State::ScriptDataEscaped;
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter
- .emit_error(Error::EofInScriptHtmlCommentLikeText);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.state = State::ScriptDataEscaped;
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscapedLessThanSign => match self.read_char()? {
- Some('/') => {
- self.temporary_buffer.clear();
- self.state = State::ScriptDataEscapedEndTagOpen;
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.temporary_buffer.clear();
- self.emitter.emit_string("<");
- self.state = State::ScriptDataDoubleEscapeStart;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("<");
- self.state = State::ScriptDataEscaped;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscapedEndTagOpen => match self.read_char()? {
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.init_end_tag();
- self.state = State::ScriptDataEscapedEndTagName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.unread_char(c);
- self.state = State::ScriptDataEscaped;
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataEscapedEndTagName => match self.read_char()? {
- Some(whitespace_pat!()) if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::BeforeAttributeName;
- Ok(ControlToken::Continue)
- }
- Some('/') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::SelfClosingStartTag;
- Ok(ControlToken::Continue)
- }
- Some('>') if self.emitter.current_is_appropriate_end_tag_token() => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.emitter.push_tag_name(ctostr!(x.to_ascii_lowercase()));
- self.temporary_buffer.push(x);
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("</");
- self.flush_buffer_characters();
- self.state = State::ScriptDataEscaped;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataDoubleEscapeStart => match self.read_char()? {
- Some(x @ whitespace_pat!() | x @ '/' | x @ '>') => {
- if self.temporary_buffer == "script" {
- self.state = State::ScriptDataDoubleEscaped;
- } else {
- self.state = State::ScriptDataEscaped;
- }
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.temporary_buffer.push(x.to_ascii_lowercase());
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- c => {
- self.state = State::ScriptDataEscaped;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataDoubleEscaped => match self.read_char()? {
- Some('-') => {
- self.state = State::ScriptDataDoubleEscapedDash;
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.state = State::ScriptDataDoubleEscapedLessThanSign;
- self.emitter.emit_string("<");
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter
- .emit_error(Error::EofInScriptHtmlCommentLikeText);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataDoubleEscapedDash => match self.read_char()? {
- Some('-') => {
- self.state = State::ScriptDataDoubleEscapedDashDash;
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.state = State::ScriptDataDoubleEscapedLessThanSign;
- self.emitter.emit_string("<");
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.state = State::ScriptDataDoubleEscaped;
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter
- .emit_error(Error::EofInScriptHtmlCommentLikeText);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.state = State::ScriptDataDoubleEscaped;
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataDoubleEscapedDashDash => match self.read_char()? {
- Some('-') => {
- self.emitter.emit_string("-");
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.emitter.emit_string("<");
- self.state = State::ScriptDataDoubleEscapedLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter.emit_string(">");
- self.state = State::ScriptData;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.state = State::ScriptDataDoubleEscaped;
- self.emitter.emit_string("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter
- .emit_error(Error::EofInScriptHtmlCommentLikeText);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.state = State::ScriptDataDoubleEscaped;
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataDoubleEscapedLessThanSign => match self.read_char()? {
- Some('/') => {
- self.temporary_buffer.clear();
- self.state = State::ScriptDataDoubleEscapeEnd;
- self.emitter.emit_string("/");
- Ok(ControlToken::Continue)
- }
- c => {
- self.state = State::ScriptDataDoubleEscaped;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::ScriptDataDoubleEscapeEnd => match self.read_char()? {
- Some(x @ whitespace_pat!() | x @ '/' | x @ '>') => {
- if self.temporary_buffer == "script" {
- self.state = State::ScriptDataEscaped;
- } else {
- self.state = State::ScriptDataDoubleEscaped;
- }
-
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- Some(x) if x.is_ascii_alphabetic() => {
- self.temporary_buffer.push(x.to_ascii_lowercase());
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- c => {
- self.state = State::ScriptDataDoubleEscaped;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::BeforeAttributeName => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- c @ Some('/' | '>') | c @ None => {
- self.state = State::AfterAttributeName;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- Some('=') => {
- self.emitter
- .emit_error(Error::UnexpectedEqualsSignBeforeAttributeName);
- self.emitter.init_attribute();
- self.emitter.push_attribute_name("=");
- self.state = State::AttributeName;
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.init_attribute();
- self.state = State::AttributeName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- },
- State::AttributeName => match self.read_char()? {
- c @ Some(whitespace_pat!() | '/' | '>') | c @ None => {
- self.state = State::AfterAttributeName;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- Some('=') => {
- self.state = State::BeforeAttributeValue;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_attribute_name("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x @ '"' | x @ '\'' | x @ '<') => {
- self.emitter
- .emit_error(Error::UnexpectedCharacterInAttributeName);
- self.emitter
- .push_attribute_name(ctostr!(x.to_ascii_lowercase()));
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter
- .push_attribute_name(ctostr!(x.to_ascii_lowercase()));
- Ok(ControlToken::Continue)
- }
- },
- State::AfterAttributeName => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('/') => {
- self.state = State::SelfClosingStartTag;
- Ok(ControlToken::Continue)
- }
- Some('=') => {
- self.state = State::BeforeAttributeValue;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInTag);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.init_attribute();
- self.state = State::AttributeName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- },
- State::BeforeAttributeValue => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('"') => {
- self.state = State::AttributeValueDoubleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('\'') => {
- self.state = State::AttributeValueSingleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter.emit_error(Error::MissingAttributeValue);
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- c => {
- self.state = State::AttributeValueUnquoted;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::AttributeValueDoubleQuoted => match self.read_char()? {
- Some('"') => {
- self.state = State::AfterAttributeValueQuoted;
- Ok(ControlToken::Continue)
- }
- Some('&') => {
- self.return_state = Some(State::AttributeValueDoubleQuoted);
- self.state = State::CharacterReference;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_attribute_value("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInTag);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_attribute_value(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::AttributeValueSingleQuoted => match self.read_char()? {
- Some('\'') => {
- self.state = State::AfterAttributeValueQuoted;
- Ok(ControlToken::Continue)
- }
- Some('&') => {
- self.return_state = Some(State::AttributeValueSingleQuoted);
- self.state = State::CharacterReference;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_attribute_value("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInTag);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_attribute_value(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::AttributeValueUnquoted => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::BeforeAttributeName;
- Ok(ControlToken::Continue)
- }
- Some('&') => {
- self.return_state = Some(State::AttributeValueUnquoted);
- self.state = State::CharacterReference;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_attribute_value("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x @ '"' | x @ '\'' | x @ '<' | x @ '=' | x @ '\u{60}') => {
- self.emitter
- .emit_error(Error::UnexpectedCharacterInUnquotedAttributeValue);
- self.emitter.push_attribute_value(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInTag);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_attribute_value(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::AfterAttributeValueQuoted => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::BeforeAttributeName;
- Ok(ControlToken::Continue)
- }
- Some('/') => {
- self.state = State::SelfClosingStartTag;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInTag);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter
- .emit_error(Error::MissingWhitespaceBetweenAttributes);
- self.state = State::BeforeAttributeName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- },
- State::SelfClosingStartTag => match self.read_char()? {
- Some('>') => {
- self.emitter.set_self_closing();
- self.state = State::Data;
- self.emitter.emit_current_tag();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInTag);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.emit_error(Error::UnexpectedSolidusInTag);
- self.state = State::BeforeAttributeName;
- self.unread_char(Some(x));
- Ok(ControlToken::Continue)
- }
- },
- State::BogusComment => match self.read_char()? {
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_comment();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_current_comment();
- Ok(ControlToken::Eof)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_comment("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some(x) => {
- self.emitter.push_comment(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::MarkupDeclarationOpen => match self.read_char()? {
- Some('-') if self.try_read_string("-", true)? => {
- self.emitter.init_comment();
- self.state = State::CommentStart;
- Ok(ControlToken::Continue)
- }
- Some('d' | 'D') if self.try_read_string("octype", false)? => {
- self.state = State::Doctype;
- Ok(ControlToken::Continue)
- }
- Some('[') if self.try_read_string("CDATA[", true)? => {
- // missing: check for adjusted current element: we don't have an element stack
- // at all
- //
- // missing: cdata transition
- //
- // let's hope that bogus comment can just sort of skip over cdata
- self.emitter.emit_error(Error::CdataInHtmlContent);
-
- self.emitter.init_comment();
- self.emitter.push_comment("[CDATA[");
- self.state = State::BogusComment;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_error(Error::IncorrectlyOpenedComment);
- self.emitter.init_comment();
- self.state = State::BogusComment;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::CommentStart => match self.read_char()? {
- Some('-') => {
- self.state = State::CommentStartDash;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter.emit_error(Error::AbruptClosingOfEmptyComment);
- self.state = State::Data;
- self.emitter.emit_current_comment();
- Ok(ControlToken::Continue)
- }
- c => {
- self.unread_char(c);
- self.state = State::Comment;
- Ok(ControlToken::Continue)
- }
- },
- State::CommentStartDash => match self.read_char()? {
- Some('-') => {
- self.state = State::CommentEnd;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter.emit_error(Error::AbruptClosingOfEmptyComment);
- self.state = State::Data;
- self.emitter.emit_current_comment();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInComment);
- self.emitter.emit_current_comment();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter.push_comment("-");
- self.unread_char(c);
- self.state = State::Comment;
- Ok(ControlToken::Continue)
- }
- },
- State::Comment => match self.read_char()? {
- Some('<') => {
- self.emitter.push_comment("<");
- self.state = State::CommentLessThanSign;
- Ok(ControlToken::Continue)
- }
- Some('-') => {
- self.state = State::CommentEndDash;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_comment("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInComment);
- self.emitter.emit_current_comment();
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_comment(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::CommentLessThanSign => match self.read_char()? {
- Some('!') => {
- self.emitter.push_comment("!");
- self.state = State::CommentLessThanSignBang;
- Ok(ControlToken::Continue)
- }
- Some('<') => {
- self.emitter.push_comment("<");
- Ok(ControlToken::Continue)
- }
- c => {
- self.unread_char(c);
- self.state = State::Comment;
- Ok(ControlToken::Continue)
- }
- },
- State::CommentLessThanSignBang => match self.read_char()? {
- Some('-') => {
- self.state = State::CommentLessThanSignBangDash;
- Ok(ControlToken::Continue)
- }
- c => {
- self.unread_char(c);
- self.state = State::Comment;
- Ok(ControlToken::Continue)
- }
- },
- State::CommentLessThanSignBangDash => match self.read_char()? {
- Some('-') => {
- self.state = State::CommentLessThanSignBangDashDash;
- Ok(ControlToken::Continue)
- }
- c => {
- self.unread_char(c);
- self.state = State::CommentEndDash;
- Ok(ControlToken::Continue)
- }
- },
- State::CommentLessThanSignBangDashDash => match self.read_char()? {
- c @ Some('>') | c @ None => {
- self.unread_char(c);
- self.state = State::CommentEnd;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_error(Error::NestedComment);
- self.unread_char(c);
- self.state = State::CommentEnd;
- Ok(ControlToken::Continue)
- }
- },
- State::CommentEndDash => match self.read_char()? {
- Some('-') => {
- self.state = State::CommentEnd;
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInComment);
- self.emitter.emit_current_comment();
- Ok(ControlToken::Eof)
- }
- c => {
- self.emitter.push_comment("-");
- self.unread_char(c);
- self.state = State::Comment;
- Ok(ControlToken::Continue)
- }
- },
- State::CommentEnd => match self.read_char()? {
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_comment();
- Ok(ControlToken::Continue)
- }
- Some('!') => {
- self.state = State::CommentEndBang;
- Ok(ControlToken::Continue)
- }
- Some('-') => {
- self.emitter.push_comment("-");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInComment);
- self.emitter.emit_current_comment();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter.push_comment("-");
- self.emitter.push_comment("-");
- self.unread_char(c);
- self.state = State::Comment;
- Ok(ControlToken::Continue)
- }
- },
- State::CommentEndBang => match self.read_char()? {
- Some('-') => {
- self.emitter.push_comment("-");
- self.emitter.push_comment("-");
- self.emitter.push_comment("!");
- self.state = State::CommentEndDash;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter.emit_error(Error::IncorrectlyClosedComment);
- self.state = State::Data;
- self.emitter.emit_current_comment();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInComment);
- self.emitter.emit_current_comment();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter.push_comment("-");
- self.emitter.push_comment("-");
- self.emitter.push_comment("!");
- self.state = State::Comment;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::Doctype => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::BeforeDoctypeName;
- Ok(ControlToken::Continue)
- }
- c @ Some('>') => {
- self.unread_char(c);
- self.state = State::BeforeDoctypeName;
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.init_doctype();
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::MissingWhitespaceBeforeDoctypeName);
- self.unread_char(c);
- self.state = State::BeforeDoctypeName;
- Ok(ControlToken::Continue)
- }
- },
- State::BeforeDoctypeName => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.init_doctype();
- self.emitter.push_doctype_name("\u{fffd}");
- self.state = State::DoctypeName;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter.emit_error(Error::MissingDoctypeName);
- self.emitter.init_doctype();
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.init_doctype();
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.init_doctype();
- self.emitter
- .push_doctype_name(ctostr!(x.to_ascii_lowercase()));
- self.state = State::DoctypeName;
- Ok(ControlToken::Continue)
- }
- },
- State::DoctypeName => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::AfterDoctypeName;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_doctype_name("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter
- .push_doctype_name(ctostr!(x.to_ascii_lowercase()));
- Ok(ControlToken::Continue)
- }
- },
- State::AfterDoctypeName => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some('p' | 'P') if self.try_read_string("ublic", false)? => {
- self.state = State::AfterDoctypePublicKeyword;
- Ok(ControlToken::Continue)
- }
- Some('s' | 'S') if self.try_read_string("ystem", false)? => {
- self.state = State::AfterDoctypeSystemKeyword;
- Ok(ControlToken::Continue)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::InvalidCharacterSequenceAfterDoctypeName);
- self.emitter.set_force_quirks();
- self.unread_char(c);
- self.state = State::BogusDoctype;
- Ok(ControlToken::Continue)
- }
- },
- State::AfterDoctypePublicKeyword => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::BeforeDoctypePublicIdentifier;
- Ok(ControlToken::Continue)
- }
- Some('"') => {
- self.emitter
- .emit_error(Error::MissingWhitespaceAfterDoctypePublicKeyword);
- self.emitter.set_doctype_public_identifier("");
- self.state = State::DoctypePublicIdentifierDoubleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('\'') => {
- self.emitter
- .emit_error(Error::MissingWhitespaceAfterDoctypePublicKeyword);
- self.emitter.set_doctype_public_identifier("");
- self.state = State::DoctypePublicIdentifierSingleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::MissingDoctypePublicIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::MissingQuoteBeforeDoctypePublicIdentifier);
- self.emitter.set_force_quirks();
- self.unread_char(c);
- self.state = State::BogusDoctype;
- Ok(ControlToken::Continue)
- }
- },
- State::BeforeDoctypePublicIdentifier => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('"') => {
- self.emitter.set_doctype_public_identifier("");
- self.state = State::DoctypePublicIdentifierDoubleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('\'') => {
- self.emitter.set_doctype_public_identifier("");
- self.state = State::DoctypePublicIdentifierSingleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::MissingDoctypePublicIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::MissingQuoteBeforeDoctypePublicIdentifier);
- self.emitter.set_force_quirks();
- self.unread_char(c);
- self.state = State::BogusDoctype;
- Ok(ControlToken::Continue)
- }
- },
- State::DoctypePublicIdentifierDoubleQuoted => match self.read_char()? {
- Some('"') => {
- self.state = State::AfterDoctypePublicIdentifier;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_doctype_public_identifier("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::AbruptDoctypePublicIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_doctype_public_identifier(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::DoctypePublicIdentifierSingleQuoted => match self.read_char()? {
- Some('\'') => {
- self.state = State::AfterDoctypePublicIdentifier;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_doctype_public_identifier("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::AbruptDoctypePublicIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_doctype_public_identifier(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::AfterDoctypePublicIdentifier => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::BetweenDoctypePublicAndSystemIdentifiers;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- Some('"') => {
- self.emitter.emit_error(
- Error::MissingWhitespaceBetweenDoctypePublicAndSystemIdentifiers,
- );
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierDoubleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('\'') => {
- self.emitter.emit_error(
- Error::MissingWhitespaceBetweenDoctypePublicAndSystemIdentifiers,
- );
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierSingleQuoted;
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::MissingQuoteBeforeDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.unread_char(c);
- self.state = State::BogusDoctype;
- Ok(ControlToken::Continue)
- }
- },
- State::BetweenDoctypePublicAndSystemIdentifiers => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- Some('"') => {
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierDoubleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('\'') => {
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierSingleQuoted;
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::MissingQuoteBeforeDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::BogusDoctype;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::AfterDoctypeSystemKeyword => match self.read_char()? {
- Some(whitespace_pat!()) => {
- self.state = State::BeforeDoctypeSystemIdentifier;
- Ok(ControlToken::Continue)
- }
- Some('"') => {
- self.emitter
- .emit_error(Error::MissingWhitespaceAfterDoctypeSystemKeyword);
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierDoubleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('\'') => {
- self.emitter
- .emit_error(Error::MissingWhitespaceAfterDoctypeSystemKeyword);
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierSingleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::MissingDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::MissingQuoteBeforeDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::BogusDoctype;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::BeforeDoctypeSystemIdentifier => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('"') => {
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierDoubleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('\'') => {
- self.emitter.set_doctype_system_identifier("");
- self.state = State::DoctypeSystemIdentifierSingleQuoted;
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::MissingDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::MissingQuoteBeforeDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::BogusDoctype;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::DoctypeSystemIdentifierDoubleQuoted => match self.read_char()? {
- Some('"') => {
- self.state = State::AfterDoctypeSystemIdentifier;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_doctype_system_identifier("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::AbruptDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_doctype_system_identifier(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::DoctypeSystemIdentifierSingleQuoted => match self.read_char()? {
- Some('\'') => {
- self.state = State::AfterDoctypeSystemIdentifier;
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- self.emitter.push_doctype_system_identifier("\u{fffd}");
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.emitter
- .emit_error(Error::AbruptDoctypeSystemIdentifier);
- self.emitter.set_force_quirks();
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.push_doctype_system_identifier(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::AfterDoctypeSystemIdentifier => match self.read_char()? {
- Some(whitespace_pat!()) => Ok(ControlToken::Continue),
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInDoctype);
- self.emitter.set_force_quirks();
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- c @ Some(_) => {
- self.emitter
- .emit_error(Error::UnexpectedCharacterAfterDoctypeSystemIdentifier);
- self.unread_char(c);
- self.state = State::BogusDoctype;
- Ok(ControlToken::Continue)
- }
- },
- State::BogusDoctype => match self.read_char()? {
- Some('>') => {
- self.state = State::Data;
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Continue)
- }
- Some('\0') => {
- self.emitter.emit_error(Error::UnexpectedNullCharacter);
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_current_doctype();
- Ok(ControlToken::Eof)
- }
- Some(_) => Ok(ControlToken::Continue),
- },
- State::CdataSection => match self.read_char()? {
- Some(']') => {
- self.state = State::CdataSectionBracket;
- Ok(ControlToken::Continue)
- }
- None => {
- self.emitter.emit_error(Error::EofInCdata);
- Ok(ControlToken::Eof)
- }
- Some(x) => {
- self.emitter.emit_string(ctostr!(x));
- Ok(ControlToken::Continue)
- }
- },
- State::CdataSectionBracket => match self.read_char()? {
- Some(']') => {
- self.state = State::CdataSectionEnd;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("]");
- self.state = State::CdataSection;
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- },
- State::CdataSectionEnd => match self.read_char()? {
- Some(']') => {
- self.emitter.emit_string("]");
- Ok(ControlToken::Continue)
- }
- Some('>') => {
- self.state = State::Data;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter.emit_string("]]");
- self.unread_char(c);
- self.state = State::CdataSection;
- Ok(ControlToken::Continue)
- }
- },
- State::CharacterReference => {
- self.temporary_buffer.clear();
- self.temporary_buffer.push('&');
- match self.read_char()? {
- Some(x) if x.is_ascii_alphanumeric() => {
- self.unread_char(Some(x));
- self.state = State::NamedCharacterReference;
- Ok(ControlToken::Continue)
- }
- Some('#') => {
- self.temporary_buffer.push('#');
- self.state = State::NumericCharacterReference;
- Ok(ControlToken::Continue)
- }
- c => {
- self.flush_code_points_consumed_as_character_reference();
- self.state = self.return_state.take().unwrap();
- self.unread_char(c);
- Ok(ControlToken::Continue)
- }
- }
- }
- State::NamedCharacterReference => {
- let c = self.read_char()?;
-
- let char_ref = match c {
- Some(x) => entities::try_read_character_reference(x, |x| {
- self.try_read_string(x, true)
- })?
- .map(|char_ref| (x, char_ref)),
-
- None => None,
- };
-
- if let Some((x, char_ref)) = char_ref {
- self.temporary_buffer.push(x);
- self.temporary_buffer.push_str(char_ref.name);
- let char_ref_name_last_character = char_ref.name.chars().last();
- let next_character = self.next_input_character()?;
- if self.is_consumed_as_part_of_an_attribute()
- && char_ref_name_last_character != Some(';')
- && matches!(next_character, Some(x) if x == '=' || x.is_ascii_alphanumeric())
- {
- self.flush_code_points_consumed_as_character_reference();
- self.state = self.return_state.take().unwrap();
- Ok(ControlToken::Continue)
- } else {
- if char_ref_name_last_character != Some(';') {
- self.emitter
- .emit_error(Error::MissingSemicolonAfterCharacterReference);
- }
-
- self.temporary_buffer.clear();
- self.temporary_buffer.push_str(char_ref.characters);
- self.flush_code_points_consumed_as_character_reference();
- self.state = self.return_state.take().unwrap();
- Ok(ControlToken::Continue)
- }
- } else {
- self.unread_char(c);
- self.flush_code_points_consumed_as_character_reference();
- self.state = State::AmbiguousAmpersand;
- Ok(ControlToken::Continue)
- }
- }
- State::AmbiguousAmpersand => match self.read_char()? {
- Some(x) if x.is_ascii_alphanumeric() => {
- if self.is_consumed_as_part_of_an_attribute() {
- self.emitter.push_attribute_value(ctostr!(x));
- } else {
- self.emitter.emit_string(ctostr!(x));
- }
-
- Ok(ControlToken::Continue)
- }
- c @ Some(';') => {
- self.emitter
- .emit_error(Error::UnknownNamedCharacterReference);
- self.unread_char(c);
- self.state = self.return_state.take().unwrap();
- Ok(ControlToken::Continue)
- }
- c => {
- self.unread_char(c);
- self.state = self.return_state.take().unwrap();
- Ok(ControlToken::Continue)
- }
- },
- State::NumericCharacterReference => {
- self.character_reference_code = 0;
- match self.read_char()? {
- Some(x @ 'x' | x @ 'X') => {
- self.temporary_buffer.push(x);
- self.state = State::HexadecimalCharacterReferenceStart;
- Ok(ControlToken::Continue)
- }
- c => {
- self.unread_char(c);
- self.state = State::DecimalCharacterReferenceStart;
- Ok(ControlToken::Continue)
- }
- }
- }
- State::HexadecimalCharacterReferenceStart => match self.read_char()? {
- c @ Some('0'..='9' | 'A'..='F' | 'a'..='f') => {
- self.unread_char(c);
- self.state = State::HexadecimalCharacterReference;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter
- .emit_error(Error::AbsenceOfDigitsInNumericCharacterReference);
- self.flush_code_points_consumed_as_character_reference();
- self.unread_char(c);
- self.state = self.return_state.take().unwrap();
- Ok(ControlToken::Continue)
- }
- },
- State::DecimalCharacterReferenceStart => match self.read_char()? {
- Some(x @ ascii_digit_pat!()) => {
- self.unread_char(Some(x));
- self.state = State::DecimalCharacterReference;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter
- .emit_error(Error::AbsenceOfDigitsInNumericCharacterReference);
- self.flush_code_points_consumed_as_character_reference();
- self.unread_char(c);
- self.state = self.return_state.take().unwrap();
- Ok(ControlToken::Continue)
- }
- },
- State::HexadecimalCharacterReference => match self.read_char()? {
- Some(x @ ascii_digit_pat!()) => {
- mutate_character_reference!(*16 + x - 0x0030);
- Ok(ControlToken::Continue)
- }
- Some(x @ 'A'..='F') => {
- mutate_character_reference!(*16 + x - 0x0037);
- Ok(ControlToken::Continue)
- }
- Some(x @ 'a'..='f') => {
- mutate_character_reference!(*16 + x - 0x0057);
- Ok(ControlToken::Continue)
- }
- Some(';') => {
- self.state = State::NumericCharacterReferenceEnd;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter
- .emit_error(Error::MissingSemicolonAfterCharacterReference);
- self.unread_char(c);
- self.state = State::NumericCharacterReferenceEnd;
- Ok(ControlToken::Continue)
- }
- },
- State::DecimalCharacterReference => match self.read_char()? {
- Some(x @ ascii_digit_pat!()) => {
- mutate_character_reference!(*10 + x - 0x0030);
- Ok(ControlToken::Continue)
- }
- Some(';') => {
- self.state = State::NumericCharacterReferenceEnd;
- Ok(ControlToken::Continue)
- }
- c => {
- self.emitter
- .emit_error(Error::MissingSemicolonAfterCharacterReference);
- self.unread_char(c);
- self.state = State::NumericCharacterReferenceEnd;
- Ok(ControlToken::Continue)
- }
- },
- State::NumericCharacterReferenceEnd => {
- match self.character_reference_code {
- 0x00 => {
- self.emitter.emit_error(Error::NullCharacterReference);
- self.character_reference_code = 0xfffd;
- }
- 0x110000.. => {
- self.emitter
- .emit_error(Error::CharacterReferenceOutsideUnicodeRange);
- self.character_reference_code = 0xfffd;
- }
- surrogate_pat!() => {
- self.emitter.emit_error(Error::SurrogateCharacterReference);
- self.character_reference_code = 0xfffd;
- }
- // noncharacter
- noncharacter_pat!() => {
- self.emitter
- .emit_error(Error::NoncharacterCharacterReference);
- }
- // 0x000d, or a control that is not whitespace
- x @ 0x000d | x @ control_pat!()
- if !matches!(x, 0x0009 | 0x000a | 0x000c | 0x0020) =>
- {
- self.emitter.emit_error(Error::ControlCharacterReference);
- self.character_reference_code = match x {
- 0x80 => 0x20AC, // EURO SIGN (€)
- 0x82 => 0x201A, // SINGLE LOW-9 QUOTATION MARK (‚)
- 0x83 => 0x0192, // LATIN SMALL LETTER F WITH HOOK (ƒ)
- 0x84 => 0x201E, // DOUBLE LOW-9 QUOTATION MARK („)
- 0x85 => 0x2026, // HORIZONTAL ELLIPSIS (…)
- 0x86 => 0x2020, // DAGGER (†)
- 0x87 => 0x2021, // DOUBLE DAGGER (‡)
- 0x88 => 0x02C6, // MODIFIER LETTER CIRCUMFLEX ACCENT (ˆ)
- 0x89 => 0x2030, // PER MILLE SIGN (‰)
- 0x8A => 0x0160, // LATIN CAPITAL LETTER S WITH CARON (Š)
- 0x8B => 0x2039, // SINGLE LEFT-POINTING ANGLE QUOTATION MARK (‹)
- 0x8C => 0x0152, // LATIN CAPITAL LIGATURE OE (Œ)
- 0x8E => 0x017D, // LATIN CAPITAL LETTER Z WITH CARON (Ž)
- 0x91 => 0x2018, // LEFT SINGLE QUOTATION MARK (‘)
- 0x92 => 0x2019, // RIGHT SINGLE QUOTATION MARK (’)
- 0x93 => 0x201C, // LEFT DOUBLE QUOTATION MARK (“)
- 0x94 => 0x201D, // RIGHT DOUBLE QUOTATION MARK (”)
- 0x95 => 0x2022, // BULLET (•)
- 0x96 => 0x2013, // EN DASH (–)
- 0x97 => 0x2014, // EM DASH (—)
- 0x98 => 0x02DC, // SMALL TILDE (˜)
- 0x99 => 0x2122, // TRADE MARK SIGN (™)
- 0x9A => 0x0161, // LATIN SMALL LETTER S WITH CARON (š)
- 0x9B => 0x203A, // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK (›)
- 0x9C => 0x0153, // LATIN SMALL LIGATURE OE (œ)
- 0x9E => 0x017E, // LATIN SMALL LETTER Z WITH CARON (ž)
- 0x9F => 0x0178, // LATIN CAPITAL LETTER Y WITH DIAERESIS (Ÿ)
- _ => self.character_reference_code,
- };
- }
- _ => (),
- }
-
- self.temporary_buffer.clear();
- self.temporary_buffer
- .push(std::char::from_u32(self.character_reference_code).unwrap());
- self.flush_code_points_consumed_as_character_reference();
- self.state = self.return_state.take().unwrap();
- Ok(ControlToken::Continue)
- }
- }
- }
-}
-
-impl<R: Reader, E: Emitter> Iterator for Tokenizer<R, E> {
- type Item = Result<E::Token, R::Error>;
-
- fn next(&mut self) -> Option<Self::Item> {
- loop {
- if let Some(token) = self.emitter.pop_token() {
- break Some(Ok(token));
- } else if !self.eof {
- match self.consume() {
- Ok(ControlToken::Continue) => (),
- Ok(ControlToken::Eof) => {
- self.eof = true;
- self.emitter.emit_eof();
- }
- Err(e) => break Some(Err(e)),
- }
- } else {
- break None;
- }
- }
- }
-}
-
-/// A kind of tokenizer that directly yields tokens when used as an iterator, so `Token` instead of
-/// `Result<Token, _>`.
-///
-/// This is the return value of [`Tokenizer::infallible`].
-pub struct InfallibleTokenizer<R: Reader<Error = Never>, E: Emitter>(Tokenizer<R, E>);
-
-impl<R: Reader<Error = Never>, E: Emitter> Tokenizer<R, E> {
- /// Statically assert that this iterator is infallible.
- ///
- /// Call this to get rid of error handling when parsing HTML from strings.
- pub fn infallible(self) -> InfallibleTokenizer<R, E> {
- InfallibleTokenizer(self)
- }
-}
-
-impl<R: Reader<Error = Never>, E: Emitter> Iterator for InfallibleTokenizer<R, E> {
- type Item = E::Token;
-
- fn next(&mut self) -> Option<Self::Item> {
- match self.0.next()? {
- Ok(token) => Some(token),
- Err(e) => match e {},
- }
- }
-}
+pub use tokenizer::{InfallibleTokenizer, Tokenizer};