diff options
| -rw-r--r-- | src/lib.rs | 1 | ||||
| -rw-r--r-- | src/tokenizer.rs | 4 | ||||
| -rw-r--r-- | src/tokenizer/machine.rs (renamed from src/machine.rs) | 0 |
3 files changed, 3 insertions, 2 deletions
@@ -11,7 +11,6 @@ mod default_emitter; mod emitter; mod entities; mod error; -mod machine; mod naive_parser; mod tokenizer; mod utils; diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 270d3d0..b46bf45 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,9 +1,11 @@ -use crate::machine::{self, ControlToken}; +mod machine; + use crate::naive_parser::naive_next_state; use crate::offset::{Offset, Position}; use crate::reader::{IntoReader, Reader}; use crate::utils::{control_pat, noncharacter_pat, surrogate_pat, State as InternalState}; use crate::{Emitter, Error}; +use machine::ControlToken; // this is a stack that can hold 0 to 2 Ts #[derive(Debug, Default, Clone, Copy)] diff --git a/src/machine.rs b/src/tokenizer/machine.rs index fd4b36b..fd4b36b 100644 --- a/src/machine.rs +++ b/src/tokenizer/machine.rs |