summaryrefslogtreecommitdiff
path: root/src/tokenizer
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizer')
-rw-r--r--src/tokenizer/machine/utils.rs13
1 files changed, 7 insertions, 6 deletions
diff --git a/src/tokenizer/machine/utils.rs b/src/tokenizer/machine/utils.rs
index d96e50b..ea4d697 100644
--- a/src/tokenizer/machine/utils.rs
+++ b/src/tokenizer/machine/utils.rs
@@ -16,7 +16,7 @@ where
/// Emits the given character as a character token.
#[inline]
pub(super) fn emit_char(&mut self, c: char) {
- self.emitter.emit_string(ctostr!(c));
+ self.emitter.emit_char(c);
}
/// Emits every byte of the given byte slice as a character token.
@@ -25,10 +25,9 @@ where
/// since [`str::chars`] isn't `const`.)
#[inline]
pub(super) fn emit_chars(&mut self, s: &[u8]) {
- self.emitter.emit_string(
- // this unsafe block is only temporary and will be removed in the next commit
- unsafe { std::str::from_utf8_unchecked(s) },
- );
+ for c in s {
+ self.emit_char(*c as char);
+ }
}
#[inline]
@@ -204,7 +203,9 @@ where
}
pub(super) fn flush_buffer_characters(&mut self) {
- self.emitter.emit_string(&self.temporary_buffer);
+ for c in self.temporary_buffer.chars() {
+ self.emitter.emit_char(c);
+ }
self.temporary_buffer.clear();
}
}