diff options
| author | Martin Fischer <martin@push-f.com> | 2023-09-09 20:38:51 +0200 |
|---|---|---|
| committer | Martin Fischer <martin@push-f.com> | 2023-09-28 10:36:08 +0200 |
| commit | a4846482707412f3cbacabeb082f25762a2636d5 (patch) | |
| tree | bbde44d58c0e1f63110e008ac3001acc11de84cf /src/tokenizer.rs | |
| parent | 595985f5e17fafedc1cf6a72691ea7eb7dc20174 (diff) | |
refactor: move machine module under tokenizer
Diffstat (limited to 'src/tokenizer.rs')
| -rw-r--r-- | src/tokenizer.rs | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 270d3d0..b46bf45 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,9 +1,11 @@ -use crate::machine::{self, ControlToken}; +mod machine; + use crate::naive_parser::naive_next_state; use crate::offset::{Offset, Position}; use crate::reader::{IntoReader, Reader}; use crate::utils::{control_pat, noncharacter_pat, surrogate_pat, State as InternalState}; use crate::{Emitter, Error}; +use machine::ControlToken; // this is a stack that can hold 0 to 2 Ts #[derive(Debug, Default, Clone, Copy)] |