1 files changed, 18 insertions, 0 deletions
diff --git a/src/tokenizer/machine/utils.rs b/src/tokenizer/machine/utils.rs
index 6e45f4d..d96e50b 100644
--- a/src/tokenizer/machine/utils.rs
+++ b/src/tokenizer/machine/utils.rs
@@ -13,6 +13,24 @@ where
     O: Offset,
     E: Emitter<O>,
 {
+    /// Emits the given character as a character token.
+    #[inline]
+    pub(super) fn emit_char(&mut self, c: char) {
+        self.emitter.emit_string(ctostr!(c));
+    }
+
+    /// Emits every byte of the given byte slice as a character token.
+    ///
+    /// (We're operating on bytes to enable compiler optimization,
+    /// since [`str::chars`] isn't `const`.)
+    #[inline]
+    pub(super) fn emit_chars(&mut self, s: &[u8]) {
+        self.emitter.emit_string(
+            // this unsafe block is only temporary and will be removed in the next commit
+            unsafe { std::str::from_utf8_unchecked(s) },
+        );
+    }
+
     #[inline]
     pub(crate) fn emit_error(&mut self, error: Error) {
         let span = match error {