diff options
Diffstat (limited to 'src/tokenizer.rs')
-rw-r--r-- | src/tokenizer.rs | 44 |
1 files changed, 1 insertions, 43 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index b2d4b53..b1dd4ba 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,10 +1,8 @@ -use std::ops::{Deref, DerefMut}; - use crate::machine; use crate::utils::{ control_pat, noncharacter_pat, surrogate_pat, ControlToken, State as InternalState, }; -use crate::{DefaultEmitter, Emitter, Error, IntoReader, Never, Reader}; +use crate::{DefaultEmitter, Emitter, Error, IntoReader, Reader}; // this is a stack that can hold 0 to 2 Ts #[derive(Debug, Default, Clone, Copy)] @@ -256,46 +254,6 @@ impl<R: Reader, E: Emitter<R>> Iterator for Tokenizer<R, E> { } } -/// A kind of tokenizer that directly yields tokens when used as an iterator, so `Token` instead of -/// `Result<Token, _>`. -/// -/// This is the return value of [`Tokenizer::infallible`]. -pub struct InfallibleTokenizer<R: Reader<Error = Never>, E: Emitter<R>>(Tokenizer<R, E>); - -impl<R: Reader<Error = Never>, E: Emitter<R>> Tokenizer<R, E> { - /// Statically assert that this iterator is infallible. - /// - /// Call this to get rid of error handling when parsing HTML from strings. - pub fn infallible(self) -> InfallibleTokenizer<R, E> { - InfallibleTokenizer(self) - } -} - -impl<R: Reader<Error = Never>, E: Emitter<R>> Iterator for InfallibleTokenizer<R, E> { - type Item = E::Token; - - fn next(&mut self) -> Option<Self::Item> { - match self.0.next()? { - Ok(token) => Some(token), - Err(e) => match e {}, - } - } -} - -impl<R: Reader<Error = Never>, E: Emitter<R>> Deref for InfallibleTokenizer<R, E> { - type Target = Tokenizer<R, E>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl<R: Reader<Error = Never>, E: Emitter<R>> DerefMut for InfallibleTokenizer<R, E> { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - impl<S: crate::spans::Span<R>, R: Reader> Tokenizer<R, DefaultEmitter<R, S>> { /// Test-internal function to override internal state. /// |