diff options
| author | Martin Fischer <martin@push-f.com> | 2023-08-28 05:40:49 +0200 |
|---|---|---|
| committer | Martin Fischer <martin@push-f.com> | 2023-09-03 23:00:05 +0200 |
| commit | f2ff370d7b9920c89bcc50cf35b7cd5bc4d2c777 (patch) | |
| tree | 910959c38d9d744ed82d4204b383d8ceb559c873 /src/tokenizer.rs | |
| parent | 7cd148aa33a0de7ae5629a51db9157f66db9ac67 (diff) | |
chore: move ControlToken enum definition to machine
Diffstat (limited to 'src/tokenizer.rs')
| -rw-r--r-- | src/tokenizer.rs | 6 |
1 files changed, 2 insertions, 4 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 05c1165..6384544 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,10 +1,8 @@ -use crate::machine; +use crate::machine::{self, ControlToken}; use crate::naive_parser::naive_next_state; use crate::offset::{Offset, Position}; use crate::reader::{IntoReader, Reader}; -use crate::utils::{ - control_pat, noncharacter_pat, surrogate_pat, ControlToken, State as InternalState, -}; +use crate::utils::{control_pat, noncharacter_pat, surrogate_pat, State as InternalState}; use crate::{Emitter, Error}; // this is a stack that can hold 0 to 2 Ts |