aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMartin Fischer <martin@push-f.com>2023-08-11 21:25:21 +0200
committerMartin Fischer <martin@push-f.com>2023-08-19 06:41:55 +0200
commit83144505291319395c1ba40035cf933786bf3422 (patch)
treef7b9cb85c8000ab3690b1813e4ee0f020dba5873
parent410bff8467a29eae8a7fba26176b8e8d25b734a2 (diff)
break!: remove InfallibleTokenizer in favor of Iterator::flatten
-rw-r--r--README.md2
-rw-r--r--src/lib.rs2
-rw-r--r--src/never.rs3
-rw-r--r--src/tokenizer.rs44
-rw-r--r--tests/test_spans.rs2
5 files changed, 4 insertions, 49 deletions
diff --git a/README.md b/README.md
index a97abb5..8305537 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ use html5tokenizer::{Tokenizer, Token};
let html = "<title >hello world</title>";
let mut new_html = String::new();
-for token in Tokenizer::new(html).infallible() {
+for token in Tokenizer::new(html).flatten() {
match token {
Token::StartTag(tag) => {
write!(new_html, "<{}>", tag.name).unwrap();
diff --git a/src/lib.rs b/src/lib.rs
index 7945a3b..27ce00f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -20,4 +20,4 @@ pub use emitter::{Attribute, DefaultEmitter, Doctype, Emitter, EndTag, StartTag,
pub use error::Error;
pub use never::Never;
pub use reader::{BufReadReader, IntoReader, Reader, StringReader};
-pub use tokenizer::{InfallibleTokenizer, State, Tokenizer};
+pub use tokenizer::{State, Tokenizer};
diff --git a/src/never.rs b/src/never.rs
index b4bb371..85a1243 100644
--- a/src/never.rs
+++ b/src/never.rs
@@ -5,9 +5,6 @@ use std::fmt;
///
/// This is used as the error type in situations where there can't be an error. A `Result<T, Never>`
/// can be safely unwrapped and the `unwrap()` may be optimized away entirely.
-///
-/// This error is typically encountered when attempting to get tokens from the `Tokenizer`. Call
-/// [`Tokenizer::infallible`] if you wish to avoid unwrapping those results yourself.
pub enum Never {}
impl fmt::Display for Never {
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index b2d4b53..b1dd4ba 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -1,10 +1,8 @@
-use std::ops::{Deref, DerefMut};
-
use crate::machine;
use crate::utils::{
control_pat, noncharacter_pat, surrogate_pat, ControlToken, State as InternalState,
};
-use crate::{DefaultEmitter, Emitter, Error, IntoReader, Never, Reader};
+use crate::{DefaultEmitter, Emitter, Error, IntoReader, Reader};
// this is a stack that can hold 0 to 2 Ts
#[derive(Debug, Default, Clone, Copy)]
@@ -256,46 +254,6 @@ impl<R: Reader, E: Emitter<R>> Iterator for Tokenizer<R, E> {
}
}
-/// A kind of tokenizer that directly yields tokens when used as an iterator, so `Token` instead of
-/// `Result<Token, _>`.
-///
-/// This is the return value of [`Tokenizer::infallible`].
-pub struct InfallibleTokenizer<R: Reader<Error = Never>, E: Emitter<R>>(Tokenizer<R, E>);
-
-impl<R: Reader<Error = Never>, E: Emitter<R>> Tokenizer<R, E> {
- /// Statically assert that this iterator is infallible.
- ///
- /// Call this to get rid of error handling when parsing HTML from strings.
- pub fn infallible(self) -> InfallibleTokenizer<R, E> {
- InfallibleTokenizer(self)
- }
-}
-
-impl<R: Reader<Error = Never>, E: Emitter<R>> Iterator for InfallibleTokenizer<R, E> {
- type Item = E::Token;
-
- fn next(&mut self) -> Option<Self::Item> {
- match self.0.next()? {
- Ok(token) => Some(token),
- Err(e) => match e {},
- }
- }
-}
-
-impl<R: Reader<Error = Never>, E: Emitter<R>> Deref for InfallibleTokenizer<R, E> {
- type Target = Tokenizer<R, E>;
-
- fn deref(&self) -> &Self::Target {
- &self.0
- }
-}
-
-impl<R: Reader<Error = Never>, E: Emitter<R>> DerefMut for InfallibleTokenizer<R, E> {
- fn deref_mut(&mut self) -> &mut Self::Target {
- &mut self.0
- }
-}
-
impl<S: crate::spans::Span<R>, R: Reader> Tokenizer<R, DefaultEmitter<R, S>> {
/// Test-internal function to override internal state.
///
diff --git a/tests/test_spans.rs b/tests/test_spans.rs
index 61b70ce..2d32260 100644
--- a/tests/test_spans.rs
+++ b/tests/test_spans.rs
@@ -25,7 +25,7 @@ fn test() {
},
DefaultEmitter::<PosTracker<StringReader>, Range<usize>>::default(),
)
- .infallible()
+ .flatten()
{
if let Token::StartTag(tag) = token {
if tag.name == "h1" {