From ad6ac5f0a825775c231e76cdc9016e61e54f4141 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Tue, 12 Sep 2023 08:19:00 +0200 Subject: break!: rename DefaultEmitter to TracingEmitter --- CHANGELOG.md | 4 +- examples/tokenize.rs | 4 +- integration_tests/tests/test_html5lib.rs | 8 +- src/default_emitter.rs | 344 ------------------------------- src/lib.rs | 4 +- src/naive_parser.rs | 8 +- src/tokenizer.rs | 2 +- src/tracing_emitter.rs | 344 +++++++++++++++++++++++++++++++ tests/test_spans.rs | 2 +- 9 files changed, 361 insertions(+), 359 deletions(-) delete mode 100644 src/default_emitter.rs create mode 100644 src/tracing_emitter.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f7452f..06831c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,15 @@ * Removed the `Error` variant. (Errors now have to be queried separately with - `DefaultEmitter::drain_errors`.) + `TracingEmitter::drain_errors`.) * Replaced the `String` variant with a new `Char` variant. (The tokenizer now emits chars instead of strings.) * Added the `EndOfFile` variant. +* The `DefaultEmitter` has been renamed to `TracingEmitter`. + * The `DefaultEmitter` now emits `Token::EndOfFile` on the end-of-file. (Previously it did not emit any token symbolizing the end-of-file.) diff --git a/examples/tokenize.rs b/examples/tokenize.rs index f8859e4..791db0f 100644 --- a/examples/tokenize.rs +++ b/examples/tokenize.rs @@ -1,13 +1,13 @@ //! Let's you easily try out the tokenizer with e.g. //! printf '

Hello world!

' | cargo run --example=tokenize -use html5tokenizer::{DefaultEmitter, Tokenizer}; +use html5tokenizer::{Tokenizer, TracingEmitter}; use std::io::BufReader; fn main() { let mut tokenizer = Tokenizer::new( BufReader::new(std::io::stdin().lock()), - DefaultEmitter::default(), + TracingEmitter::default(), ); while let Some(token) = tokenizer.next() { for (error, _) in tokenizer.emitter_mut().drain_errors() { diff --git a/integration_tests/tests/test_html5lib.rs b/integration_tests/tests/test_html5lib.rs index a682cb3..36fb880 100644 --- a/integration_tests/tests/test_html5lib.rs +++ b/integration_tests/tests/test_html5lib.rs @@ -6,7 +6,7 @@ use html5lib_tests::{ use html5tokenizer::{ offset::{Offset, Position}, reader::Reader, - CdataAction, DefaultEmitter, Emitter, Error, Event, InternalState, Token, Tokenizer, + CdataAction, Emitter, Error, Event, InternalState, Token, Tokenizer, TracingEmitter, }; use similar_asserts::assert_eq; @@ -73,7 +73,7 @@ fn run_test(fname: &str, test_i: usize, test: Test) { test_i, &test, state, - Tokenizer::new(&test.input, DefaultEmitter::default()), + Tokenizer::new(&test.input, TracingEmitter::default()), "string", ); @@ -84,7 +84,7 @@ fn run_test(fname: &str, test_i: usize, test: Test) { state, Tokenizer::new( BufReader::new(test.input.as_bytes()), - DefaultEmitter::default(), + TracingEmitter::default(), ), "bufread", ); @@ -181,7 +181,7 @@ trait DrainErrors { fn drain_errors(&mut self) -> Box)> + '_>; } -impl DrainErrors for DefaultEmitter { +impl DrainErrors for TracingEmitter { fn drain_errors(&mut self) -> Box)> + '_> { Box::new(self.drain_errors()) } diff --git a/src/default_emitter.rs b/src/default_emitter.rs deleted file mode 100644 index 7b6c51e..0000000 --- a/src/default_emitter.rs +++ /dev/null @@ -1,344 +0,0 @@ -use std::collections::btree_map::Entry; -use std::collections::BTreeSet; -use std::collections::VecDeque; -use std::ops::Range; - -use crate::let_else::assume; -use crate::offset::NoopOffset; -use crate::offset::Offset; -use crate::token::{AttrValueSyntax, Comment, Doctype, EndTag, StartTag, Token}; -use crate::Emitter; -use crate::Error; - -/// The default implementation of [`Emitter`], used to produce tokens. -pub struct DefaultEmitter { - current_token: Option>, - current_attribute_name: String, - current_attr_internal: crate::token::AttrInternal, - seen_attributes: BTreeSet, - emitted_tokens: VecDeque>, - errors: VecDeque<(Error, Range)>, - attr_in_end_tag_span: Option>, -} - -impl Default for DefaultEmitter { - fn default() -> Self { - DefaultEmitter { - current_token: None, - current_attribute_name: String::new(), - current_attr_internal: Default::default(), - seen_attributes: BTreeSet::new(), - emitted_tokens: VecDeque::new(), - errors: VecDeque::new(), - attr_in_end_tag_span: None, - } - } -} - -impl DefaultEmitter { - /// Removes all encountered tokenizer errors and returns them as an iterator. - pub fn drain_errors(&mut self) -> impl Iterator)> + '_ { - self.errors.drain(0..) - } -} - -impl Iterator for DefaultEmitter { - type Item = Token; - - fn next(&mut self) -> Option { - self.emitted_tokens.pop_back() - } -} - -impl Emitter for DefaultEmitter { - fn report_error(&mut self, error: Error, span: Range) { - self.errors.push_back((error, span)); - } - - fn emit_char(&mut self, c: char) { - self.emit_token(Token::Char(c)); - } - - fn emit_eof(&mut self) { - self.emit_token(Token::EndOfFile); - } - - fn init_start_tag(&mut self, tag_offset: O, name_offset: O) { - self.current_token = Some(Token::StartTag(StartTag { - span: tag_offset..O::default(), - self_closing: false, - name: String::new(), - attributes: Default::default(), - name_span: name_offset..O::default(), - })); - } - - fn init_end_tag(&mut self, tag_offset: O, name_offset: O) { - self.current_token = Some(Token::EndTag(EndTag { - span: tag_offset..O::default(), - name: String::new(), - name_span: name_offset..O::default(), - })); - self.seen_attributes.clear(); - } - - fn push_tag_name(&mut self, s: &str) { - assume!( - Some(Token::StartTag(StartTag { name, .. }) | Token::EndTag(EndTag { name, .. })), - &mut self.current_token - ); - name.push_str(s); - } - - fn terminate_tag_name(&mut self, offset: O) { - assume!( - Some( - Token::StartTag(StartTag { name_span, .. }) - | Token::EndTag(EndTag { name_span, .. }) - ), - &mut self.current_token - ); - name_span.end = offset; - } - - fn init_attribute_name(&mut self, offset: O) { - self.flush_current_attribute(); - self.current_attr_internal.name_span.start = offset; - } - - fn push_attribute_name(&mut self, s: &str) { - self.current_attribute_name.push_str(s); - } - - fn terminate_attribute_name(&mut self, offset: O) { - self.current_attr_internal.name_span.end = offset; - } - - fn init_attribute_value(&mut self, syntax: AttrValueSyntax, offset: O) { - self.current_attr_internal.value_span.start = offset; - self.current_attr_internal.value_syntax = Some(syntax); - } - - fn push_attribute_value(&mut self, s: &str) { - self.current_attr_internal.value.push_str(s); - } - - fn terminate_attribute_value(&mut self, offset: O) { - self.current_attr_internal.value_span.end = offset; - } - - fn set_self_closing(&mut self, slash_span: Range) { - let token = self.current_token.as_mut().unwrap(); - - match token { - Token::StartTag(tag) => { - tag.self_closing = true; - } - Token::EndTag(_) => { - self.report_error(Error::EndTagWithTrailingSolidus, slash_span); - } - other => debug_assert!(false, "unexpected current_token: {other:?}"), - } - } - - fn emit_current_tag(&mut self, offset: O) { - self.flush_current_attribute(); - let mut token = self.current_token.take().unwrap(); - match &mut token { - Token::EndTag(tag) => { - if !self.seen_attributes.is_empty() { - let span = self.attr_in_end_tag_span.take().unwrap(); - self.report_error(Error::EndTagWithAttributes, span); - } - self.seen_attributes.clear(); - tag.span.end = offset; - } - Token::StartTag(tag) => { - tag.span.end = offset; - } - other => { - debug_assert!(false, "unexpected current_token: {other:?}"); - return; - } - } - self.emit_token(token); - } - - fn init_comment(&mut self, data_start_offset: O) { - self.current_token = Some(Token::Comment(Comment { - data: String::new(), - data_span: data_start_offset..O::default(), - })); - } - - fn push_comment(&mut self, s: &str) { - assume!(Some(Token::Comment(comment)), &mut self.current_token); - comment.data.push_str(s); - } - - fn emit_current_comment(&mut self, data_end_offset: O) { - let mut token = self.current_token.take().unwrap(); - assume!(Token::Comment(comment), &mut token); - comment.data_span.end = data_end_offset; - self.emit_token(token); - } - - fn init_doctype(&mut self, offset: O) { - self.current_token = Some(Token::Doctype(Doctype { - name: None, - force_quirks: false, - public_id: None, - system_id: None, - span: offset..O::default(), - name_span: O::default()..O::default(), - public_id_span: O::default()..O::default(), - system_id_span: O::default()..O::default(), - })); - } - - fn init_doctype_name(&mut self, offset: O) { - assume!(Some(Token::Doctype(doctype)), &mut self.current_token); - doctype.name = Some("".into()); - doctype.name_span.start = offset; - } - - fn push_doctype_name(&mut self, s: &str) { - assume!( - Some(Token::Doctype(Doctype { - name: Some(name), - .. - })), - &mut self.current_token - ); - name.push_str(s); - } - - fn terminate_doctype_name(&mut self, offset: O) { - assume!(Some(Token::Doctype(doctype)), &mut self.current_token); - doctype.name_span.end = offset; - } - - fn init_doctype_public_id(&mut self, offset: O) { - assume!(Some(Token::Doctype(doctype)), &mut self.current_token); - doctype.public_id = Some("".to_owned()); - doctype.public_id_span.start = offset; - } - - fn push_doctype_public_id(&mut self, s: &str) { - assume!( - Some(Token::Doctype(Doctype { - public_id: Some(public_id), - .. - })), - &mut self.current_token - ); - public_id.push_str(s); - } - - fn terminate_doctype_public_id(&mut self, offset: O) { - assume!(Some(Token::Doctype(doctype)), &mut self.current_token); - doctype.public_id_span.end = offset; - } - - fn init_doctype_system_id(&mut self, offset: O) { - assume!(Some(Token::Doctype(doctype)), &mut self.current_token); - doctype.system_id = Some("".to_owned()); - doctype.system_id_span.start = offset; - } - - fn push_doctype_system_id(&mut self, s: &str) { - assume!( - Some(Token::Doctype(Doctype { - system_id: Some(id), - .. - })), - &mut self.current_token - ); - id.push_str(s); - } - - fn terminate_doctype_system_id(&mut self, offset: O) { - assume!(Some(Token::Doctype(doctype)), &mut self.current_token); - doctype.system_id_span.end = offset; - } - - fn set_force_quirks(&mut self) { - assume!(Some(Token::Doctype(doctype)), &mut self.current_token); - doctype.force_quirks = true; - } - - fn emit_current_doctype(&mut self, offset: O) { - assume!(Some(Token::Doctype(mut doctype)), self.current_token.take()); - doctype.span.end = offset; - self.emit_token(Token::Doctype(doctype)); - } -} - -impl DefaultEmitter { - fn emit_token(&mut self, token: Token) { - self.emitted_tokens.push_front(token); - } - - fn flush_current_attribute(&mut self) - where - O: Offset, - { - if self.current_attribute_name.is_empty() { - return; - } - let name = std::mem::take(&mut self.current_attribute_name); - let attr_internal = std::mem::take(&mut self.current_attr_internal); - - match &mut self.current_token { - Some(Token::StartTag(tag)) => match tag.attributes.inner.entry(name) { - Entry::Vacant(vacant) => { - vacant.insert(attr_internal); - } - Entry::Occupied(_) => { - self.report_error(Error::DuplicateAttribute, attr_internal.name_span); - } - }, - Some(Token::EndTag(_)) => { - self.attr_in_end_tag_span = Some(attr_internal.name_span.clone()); - if !self.seen_attributes.insert(name) { - self.report_error(Error::DuplicateAttribute, attr_internal.name_span); - } - } - other => debug_assert!(false, "unexpected current_token: {other:?}"), - } - } -} - -/// The majority of our testing of the [`DefaultEmitter`] is done against the -/// html5lib-tests in the html5lib integration test. This module only tests -/// details that aren't present in the html5lib test data. -#[cfg(test)] -mod tests { - use super::DefaultEmitter; - use crate::token::{AttrValueSyntax, Token}; - use crate::{Event, Tokenizer}; - - #[test] - fn test_attribute_value_syntax() { - let mut tokenizer = Tokenizer::new( - "
", - DefaultEmitter::default(), - ) - .flatten(); - let Event::Token(Token::StartTag(tag)) = tokenizer.next().unwrap() else { - panic!("expected start tag"); - }; - for (name, syntax) in [ - ("empty", None), - ("unquoted", Some(AttrValueSyntax::Unquoted)), - ("single-quoted", Some(AttrValueSyntax::SingleQuoted)), - ("double-quoted", Some(AttrValueSyntax::DoubleQuoted)), - ] { - assert_eq!( - tag.attributes.get(name).unwrap().value_syntax(), - syntax, - "unexpected value for attribute {name}" - ); - } - } -} diff --git a/src/lib.rs b/src/lib.rs index 40b691a..aecbef3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,13 +7,13 @@ #![doc = concat!("[the LICENSE file]: ", file_url!("LICENSE"))] #![doc = include_str!("../README.md")] -mod default_emitter; mod emitter; mod entities; mod error; mod let_else; mod naive_parser; mod tokenizer; +mod tracing_emitter; /// Types for HTML attributes. pub mod attr { @@ -25,12 +25,12 @@ pub mod offset; pub mod reader; pub mod token; -pub use default_emitter::DefaultEmitter; pub use emitter::Emitter; pub use error::Error; pub use naive_parser::NaiveParser; pub use token::{Comment, Doctype, EndTag, StartTag, Token}; pub use tokenizer::{CdataAction, Event, State, Tokenizer}; +pub use tracing_emitter::TracingEmitter; #[cfg(feature = "integration-tests")] pub use tokenizer::InternalState; diff --git a/src/naive_parser.rs b/src/naive_parser.rs index 4988477..91edbc0 100644 --- a/src/naive_parser.rs +++ b/src/naive_parser.rs @@ -1,7 +1,7 @@ -use crate::default_emitter::DefaultEmitter; use crate::offset::{Offset, Position}; use crate::reader::{IntoReader, Reader}; use crate::tokenizer::CdataAction; +use crate::tracing_emitter::TracingEmitter; use crate::{Emitter, Event, State, Tokenizer}; /// A naive HTML parser (**not** spec-compliant since it doesn't do tree construction). @@ -30,18 +30,18 @@ pub struct NaiveParser> { tokenizer: Tokenizer, } -impl NaiveParser> +impl NaiveParser> where R: Reader + Position, O: Offset, { /// Constructs a new naive parser. // TODO: add example for NaiveParser::new - pub fn new<'a, IR>(reader: IR) -> NaiveParser> + pub fn new<'a, IR>(reader: IR) -> NaiveParser> where IR: IntoReader<'a, Reader = R>, { - NaiveParser::new_with_emitter(reader, DefaultEmitter::default()) + NaiveParser::new_with_emitter(reader, TracingEmitter::default()) } } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7c38e49..d0e2eaf 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -15,7 +15,7 @@ pub use machine::State as InternalState; /// Iterating over the tokenizer directly without calling [`Tokenizer::set_state`] /// results in wrong state transitions: /// -/// ``` +/// ```ignore TODO: unignore once the BasicEmitter has been implemented /// # use html5tokenizer::{DefaultEmitter, Event, Tokenizer, Token}; /// let emitter = DefaultEmitter::default(); /// let html = "