aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--benches/html5ever.rs4
-rw-r--r--examples/noop-tokenize.rs4
-rw-r--r--examples/tokenize.rs4
-rw-r--r--src/tokenizer/interface.rs8
-rw-r--r--src/tokenizer/mod.rs47
5 files changed, 23 insertions, 44 deletions
diff --git a/benches/html5ever.rs b/benches/html5ever.rs
index 8e4bbaf..b8c89c1 100644
--- a/benches/html5ever.rs
+++ b/benches/html5ever.rs
@@ -11,9 +11,7 @@ use html5tokenizer::{BufferQueue, Token, TokenSink, TokenSinkResult, Tokenizer};
struct Sink;
impl TokenSink for Sink {
- type Handle = ();
-
- fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+ fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult {
// Don't use the token, but make sure we don't get
// optimized out entirely.
black_box(token);
diff --git a/examples/noop-tokenize.rs b/examples/noop-tokenize.rs
index 305727e..22a2192 100644
--- a/examples/noop-tokenize.rs
+++ b/examples/noop-tokenize.rs
@@ -20,9 +20,7 @@ use io::Read;
struct Sink(Vec<Token>);
impl TokenSink for Sink {
- type Handle = ();
-
- fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+ fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult {
// Don't use the token, but make sure we don't get
// optimized out entirely.
self.0.push(token);
diff --git a/examples/tokenize.rs b/examples/tokenize.rs
index d70bfff..8728a18 100644
--- a/examples/tokenize.rs
+++ b/examples/tokenize.rs
@@ -40,9 +40,7 @@ impl TokenPrinter {
}
impl TokenSink for TokenPrinter {
- type Handle = ();
-
- fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult<()> {
+ fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult {
match token {
CharacterTokens(b) => {
for c in b.chars() {
diff --git a/src/tokenizer/interface.rs b/src/tokenizer/interface.rs
index 53b06ae..2c6cc38 100644
--- a/src/tokenizer/interface.rs
+++ b/src/tokenizer/interface.rs
@@ -94,19 +94,17 @@ pub enum Token {
#[derive(Debug, PartialEq)]
#[must_use]
-pub enum TokenSinkResult<Handle> {
+pub enum TokenSinkResult {
Continue,
- Script(Handle),
+ Break,
Plaintext,
RawData(states::RawKind),
}
/// Types which can receive tokens from the tokenizer.
pub trait TokenSink {
- type Handle;
-
/// Process a token.
- fn process_token(&mut self, token: Token, line_number: u64) -> TokenSinkResult<Self::Handle>;
+ fn process_token(&mut self, token: Token, line_number: u64) -> TokenSinkResult;
// Signal sink that tokenization reached the end.
fn end(&mut self) {}
diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs
index 9cdb398..0a0f7af 100644
--- a/src/tokenizer/mod.rs
+++ b/src/tokenizer/mod.rs
@@ -37,15 +37,15 @@ mod char_ref;
mod interface;
mod states;
-pub enum ProcessResult<Handle> {
+pub enum ProcessResult {
Suspend,
- Script(Handle),
+ Break,
}
#[must_use]
-pub enum TokenizerResult<Handle> {
+pub enum TokenizerResult {
Done,
- Script(Handle),
+ Break,
}
fn option_push(opt_str: &mut Option<String>, c: char) {
@@ -197,7 +197,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
}
/// Feed an input string into the tokenizer.
- pub fn feed(&mut self, input: &mut BufferQueue) -> TokenizerResult<Sink::Handle> {
+ pub fn feed(&mut self, input: &mut BufferQueue) -> TokenizerResult {
if input.is_empty() {
return TokenizerResult::Done;
}
@@ -215,7 +215,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
self.run(input)
}
- fn process_token(&mut self, token: Token) -> TokenSinkResult<Sink::Handle> {
+ fn process_token(&mut self, token: Token) -> TokenSinkResult {
if self.opts.profile {
let (ret, dt) = time!(self.sink.process_token(token, self.current_line));
self.time_in_sink += dt;
@@ -325,7 +325,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
}
/// Run the state machine for as long as we can.
- fn run(&mut self, input: &mut BufferQueue) -> TokenizerResult<Sink::Handle> {
+ fn run(&mut self, input: &mut BufferQueue) -> TokenizerResult {
if self.opts.profile {
loop {
let state = self.state;
@@ -346,9 +346,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
match run {
ControlFlow::Continue(()) => (),
ControlFlow::Break(ProcessResult::Suspend) => break,
- ControlFlow::Break(ProcessResult::Script(node)) => {
- return TokenizerResult::Script(node)
- }
+ ControlFlow::Break(ProcessResult::Break) => return TokenizerResult::Break,
}
}
} else {
@@ -356,9 +354,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
match self.step(input) {
ControlFlow::Continue(()) => (),
ControlFlow::Break(ProcessResult::Suspend) => break,
- ControlFlow::Break(ProcessResult::Script(node)) => {
- return TokenizerResult::Script(node)
- }
+ ControlFlow::Break(ProcessResult::Break) => return TokenizerResult::Break,
}
}
}
@@ -398,7 +394,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
self.process_token_and_continue(CharacterTokens(b));
}
- fn emit_current_tag(&mut self) -> ControlFlow<ProcessResult<Sink::Handle>> {
+ fn emit_current_tag(&mut self) -> ControlFlow<ProcessResult> {
self.finish_attribute();
let name = self.current_tag_name.clone();
@@ -445,9 +441,9 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
self.state = states::Plaintext;
ControlFlow::Continue(())
}
- TokenSinkResult::Script(node) => {
+ TokenSinkResult::Break => {
self.state = states::Data;
- ControlFlow::Break(ProcessResult::Script(node))
+ ControlFlow::Break(ProcessResult::Break)
}
TokenSinkResult::RawData(kind) => {
self.state = states::RawData(kind);
@@ -679,7 +675,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
// Return true if we should be immediately re-invoked
// (this just simplifies control flow vs. break / continue).
#[allow(clippy::never_loop)]
- fn step(&mut self, input: &mut BufferQueue) -> ControlFlow<ProcessResult<Sink::Handle>> {
+ fn step(&mut self, input: &mut BufferQueue) -> ControlFlow<ProcessResult> {
if self.char_ref_tokenizer.is_some() {
return self.step_char_ref_tokenizer(input);
}
@@ -1897,10 +1893,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
}
}
- fn step_char_ref_tokenizer(
- &mut self,
- input: &mut BufferQueue,
- ) -> ControlFlow<ProcessResult<Sink::Handle>> {
+ fn step_char_ref_tokenizer(&mut self, input: &mut BufferQueue) -> ControlFlow<ProcessResult> {
// FIXME HACK: Take and replace the tokenizer so we don't
// double-mut-borrow self. This is why it's boxed.
let mut tok = self.char_ref_tokenizer.take().unwrap();
@@ -1973,7 +1966,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
match self.eof_step() {
ControlFlow::Continue(()) => (),
ControlFlow::Break(ProcessResult::Suspend) => break,
- ControlFlow::Break(ProcessResult::Script(_)) => unreachable!(),
+ ControlFlow::Break(ProcessResult::Break) => unreachable!(),
}
}
@@ -2003,7 +1996,7 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
}
}
- fn eof_step(&mut self) -> ControlFlow<ProcessResult<Sink::Handle>> {
+ fn eof_step(&mut self) -> ControlFlow<ProcessResult> {
match self.state {
states::Data
| states::RawData(Rcdata)
@@ -2191,17 +2184,11 @@ mod test {
}
impl TokenSink for LinesMatch {
- type Handle = ();
-
fn end(&mut self) {
self.finish_str();
}
- fn process_token(
- &mut self,
- token: Token,
- line_number: u64,
- ) -> TokenSinkResult<Self::Handle> {
+ fn process_token(&mut self, token: Token, line_number: u64) -> TokenSinkResult {
match token {
CharacterTokens(b) => {
self.current_str_line = line_number;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 09:34:22 +0000