From f588704c90f33fe27945d742762d016dea3e113c Mon Sep 17 00:00:00 2001
From: Martin Fischer <martin@push-f.com>
Date: Fri, 1 Sep 2023 23:00:45 +0200
Subject: fix: don't assume UTF-8 in machine/tokenizer
---
src/machine.rs | 47 ++++++++++++++++++++++++++++++++++-------------
src/tokenizer.rs | 5 ++++-
2 files changed, 38 insertions(+), 14 deletions(-)
(limited to 'src')
diff --git a/src/machine.rs b/src/machine.rs
index f7f5ac6..5b36eee 100644
--- a/src/machine.rs
+++ b/src/machine.rs
@@ -38,6 +38,8 @@ where
};
}
+ slf.position_before_match = slf.reader.position();
+
match slf.state {
State::Data => match slf.read_char()? {
Some('&') => {
@@ -46,7 +48,7 @@ where
Ok(ControlToken::Continue)
}
Some('<') => {
- slf.some_offset = slf.reader.position() - 1;
+ slf.some_offset = slf.position_before_match;
slf.state = State::TagOpen;
Ok(ControlToken::Continue)
}
@@ -702,7 +704,7 @@ where
Ok(ControlToken::Continue)
}
Some(x) => {
- slf.emitter.init_attribute_name(slf.reader.position() - 1);
+ slf.emitter.init_attribute_name(slf.position_before_match);
slf.state = State::AttributeName;
slf.unread_char(Some(x));
Ok(ControlToken::Continue)
@@ -755,7 +757,7 @@ where
Ok(ControlToken::Eof)
}
Some(x) => {
- slf.emitter.init_attribute_name(slf.reader.position() - 1);
+ slf.emitter.init_attribute_name(slf.position_before_match);
slf.state = State::AttributeName;
slf.unread_char(Some(x));
Ok(ControlToken::Continue)
@@ -783,7 +785,7 @@ where
}
c => {
slf.emitter
- .init_attribute_value(AttrValueSyntax::Unquoted, slf.reader.position() - 1);
+ .init_attribute_value(AttrValueSyntax::Unquoted, slf.position_before_match);
slf.state = State::AttributeValueUnquoted;
slf.unread_char(c);
Ok(ControlToken::Continue)
@@ -791,8 +793,13 @@ where
},
State::AttributeValueDoubleQuoted => match slf.read_char()? {
Some('"') => {
- slf.emitter
- .terminate_attribute_value(slf.reader.position() - 1);
+ slf.emitter.terminate_attribute_value(
+ // We cannot simply pass slf.position_before_match because
+ // State::NamedCharacterReference calls Tokenizer::unread_char
+ // which Reader::position doesn't account for.
+ // TODO: pass slf.position_before_match once CharacterReference has been converted to a function call
+ slf.reader.position() - slf.reader.len_of_char_in_current_encoding('"'),
+ );
slf.state = State::AfterAttributeValueQuoted;
Ok(ControlToken::Continue)
}
@@ -817,8 +824,13 @@ where
},
State::AttributeValueSingleQuoted => match slf.read_char()? {
Some('\'') => {
- slf.emitter
- .terminate_attribute_value(slf.reader.position() - 1);
+ slf.emitter.terminate_attribute_value(
+ // We cannot simply pass slf.position_before_match because
+ // State::NamedCharacterReference calls Tokenizer::unread_char
+ // which Reader::position doesn't account for.
+ // TODO: pass slf.position_before_match once CharacterReference has been converted to a function call
+ slf.reader.position() - slf.reader.len_of_char_in_current_encoding('\''),
+ );
slf.state = State::AfterAttributeValueQuoted;
Ok(ControlToken::Continue)
}
@@ -843,8 +855,13 @@ where
},
State::AttributeValueUnquoted => match slf.read_char()? {
Some(whitespace_pat!()) => {
- slf.emitter
- .terminate_attribute_value(slf.reader.position() - 1);
+ slf.emitter.terminate_attribute_value(
+ // We cannot simply pass slf.position_before_match because
+ // State::NamedCharacterReference calls Tokenizer::unread_char
+ // which Reader::position doesn't account for.
+ // TODO: pass slf.position_before_match once CharacterReference has been converted to a function call
+ slf.reader.position() - slf.reader.len_of_char_in_current_encoding(' '),
+ );
slf.state = State::BeforeAttributeName;
Ok(ControlToken::Continue)
}
@@ -904,7 +921,9 @@ where
},
State::SelfClosingStartTag => match slf.read_char()? {
Some('>') => {
- slf.emitter.set_self_closing(slf.reader.position() - 2);
+ slf.emitter.set_self_closing(
+ slf.position_before_match - slf.reader.len_of_char_in_current_encoding('/'),
+ );
slf.state = State::Data;
slf.emit_current_tag();
Ok(ControlToken::Continue)
@@ -953,7 +972,7 @@ where
Some('[') if slf.try_read_string("CDATA[", true)? => Ok(ControlToken::CdataOpen),
c => {
slf.emit_error(Error::IncorrectlyOpenedComment);
- slf.emitter.init_comment(slf.reader.position() - 1);
+ slf.emitter.init_comment(slf.position_before_match);
slf.state = State::BogusComment;
slf.unread_char(c);
Ok(ControlToken::Continue)
@@ -1659,7 +1678,9 @@ where
}
},
State::CharacterReference => {
- slf.some_offset = slf.reader.position() - "&".len();
+ // TODO: we can avoid these Reader method calls by changing CharacterReference to be a function instead of a state
+ slf.some_offset =
+ slf.reader.position() - slf.reader.len_of_char_in_current_encoding('&');
slf.temporary_buffer.clear();
slf.temporary_buffer.push('&');
match slf.read_char()? {
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index d59710d..58f7b80 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -49,6 +49,8 @@ pub struct Tokenizer<R: Reader, O, E: Emitter<O>> {
current_tag_name: String,
last_start_tag_name: String,
is_start_tag: bool,
+ /// The reader position before the match block in [`machine::consume`].
+ pub(crate) position_before_match: O,
/// * Set to the offset of `<` in [`InternalState::Data`].
/// * Set to the offset of `&` in [`InternalState::CharacterReference`].
pub(crate) some_offset: O,
@@ -77,6 +79,7 @@ impl<R: Reader + Position<O>, O: Offset, E: Emitter<O>> Tokenizer<R, O, E> {
current_tag_name: String::new(),
last_start_tag_name: String::new(),
is_start_tag: false,
+ position_before_match: O::default(),
some_offset: O::default(),
naively_switch_state: false,
}
@@ -198,7 +201,7 @@ impl<R: Reader + Position<O>, O: Offset, E: Emitter<O>> Tokenizer<R, O, E> {
| Error::ControlCharacterReference
| Error::UnknownNamedCharacterReference => self.some_offset..self.reader.position(),
- _ => self.reader.position() - 1..self.reader.position(),
+ _ => self.position_before_match..self.reader.position(),
};
self.emitter.emit_error(error, span);
}
--
cgit v1.2.3