diff options
Diffstat (limited to 'src/tokenizer.rs')
| -rw-r--r-- | src/tokenizer.rs | 3 |
1 files changed, 0 insertions, 3 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index b1dd4ba..d7db3b6 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -44,9 +44,6 @@ pub struct Tokenizer<R: Reader, E: Emitter<R> = DefaultEmitter<R, ()>> { impl<R: Reader> Tokenizer<R> { /// Create a new tokenizer from some input. - /// - /// Patches are welcome for providing an efficient implementation over async streams, - /// iterators, files, etc, as long as any dependencies come behind featureflags. pub fn new<'a, S: IntoReader<'a, Reader = R>>(input: S) -> Self { Tokenizer::<S::Reader>::new_with_emitter(input, DefaultEmitter::default()) } |