aboutsummaryrefslogtreecommitdiff
path: root/src/reader.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/reader.rs')
-rw-r--r--src/reader.rs135
1 files changed, 109 insertions, 26 deletions
diff --git a/src/reader.rs b/src/reader.rs
index a987b28..15b9224 100644
--- a/src/reader.rs
+++ b/src/reader.rs
@@ -1,7 +1,8 @@
//! Provides the [`Reader`] trait (and implementations) used by the tokenizer.
+use std::collections::VecDeque;
use std::convert::Infallible;
-use std::io::{self, BufRead, BufReader, Read};
+use std::io::{self, BufReader, Read};
/// An object that provides characters to the tokenizer.
///
@@ -27,6 +28,7 @@ pub trait Reader {
///
/// If the next characters equal to `s`, this function consumes the respective characters from
/// the input stream and returns `true`. If not, it does nothing and returns `false`.
+ // TODO: document a maximum s length that may be assumed (depends on longest named character reference ... which may change?)
fn try_read_string(&mut self, s: &str, case_sensitive: bool) -> Result<bool, Self::Error>;
}
@@ -112,57 +114,139 @@ impl<'a> IntoReader<'a> for &'a String {
}
}
-/// A [`BufRead`]-based [`Reader`] implementation that attempts to read UTF-8.
-pub struct BufReadReader<R: BufRead> {
- line: String,
- line_pos: usize,
+/// Just the same as [`std::sys_common::io::DEFAULT_BUF_SIZE`] (which isn't public).
+const BUF_SIZE: usize = 8 * 1024;
+
+/// A [`Read`]-based buffered [`Reader`] implementation that attempts to read UTF-8.
+pub struct BufReadReader<R: Read> {
reader: R,
+ /// The buffer into which bytes will be read from the reader.
+ buffer: [u8; BUF_SIZE],
+ /// Number of bytes in the buffer that have been read.
+ read: usize,
+ /// Position in the buffer up until the bytes have been parsed to chars.
+ pos: usize,
+ /// The characters parsed from the buffer in read order.
+ chars: VecDeque<char>,
+ /// An error that has occurred after reading the current content of chars.
+ error: Option<io::Error>,
+ /// Indicates if the end-of-file has been reached (we won't read anymore).
+ eof: bool,
}
-impl<R: BufRead> BufReadReader<R> {
- /// Construct a new `BufReadReader` from any type that implements `BufRead`.
+impl<R: Read> BufReadReader<R> {
+ /// Construct a new `BufReadReader` from any type that implements [`Read`].
pub fn new(reader: R) -> Self {
BufReadReader {
- line: String::new(),
- line_pos: 0,
reader,
+ buffer: [0; BUF_SIZE],
+ read: 0,
+ pos: 0,
+ chars: VecDeque::new(),
+ error: None,
+ eof: false,
}
}
#[inline]
- fn get_remaining_line(&mut self) -> Result<&str, io::Error> {
- if self.line_pos < self.line.len() {
- return Ok(&self.line[self.line_pos..]);
+ fn read(&mut self) -> Result<(), io::Error> {
+ debug_assert!(!self.eof);
+ debug_assert!(self.error.is_none());
+
+ if self.pos == self.read {
+ self.read = match self.reader.read(&mut self.buffer)? {
+ 0 => {
+ self.eof = true;
+ return Ok(());
+ }
+ n => n,
+ };
+ self.pos = 0;
+ }
+
+ let unprocessed = &self.buffer[self.pos..self.read];
+
+ let (valid_str, err) = match std::str::from_utf8(unprocessed) {
+ Ok(s) => (s, None),
+ Err(err) => (
+ // SAFETY: The UTF-8 checking has already been done by the previous from_utf8 call.
+ unsafe { std::str::from_utf8_unchecked(&unprocessed[..err.valid_up_to()]) },
+ Some(err),
+ ),
+ };
+ for c in valid_str.chars() {
+ self.chars.push_back(c);
}
+ self.pos += valid_str.len();
- self.line.clear();
- self.line_pos = 0;
- self.reader.read_line(&mut self.line)?;
- Ok(&self.line)
+ if let Some(err) = err {
+ self.error = Some(io::Error::new(io::ErrorKind::InvalidData, err));
+
+ match err.error_len() {
+ None => self.eof = true,
+ Some(error_len) => self.pos += error_len,
+ }
+ }
+ Ok(())
}
}
-impl<R: BufRead> Reader for BufReadReader<R> {
+impl<R: Read> Reader for BufReadReader<R> {
type Error = io::Error;
fn read_char(&mut self) -> Result<Option<char>, Self::Error> {
- let rv = self.get_remaining_line()?.chars().next();
- self.line_pos += rv.map(char::len_utf8).unwrap_or(1);
- Ok(rv)
+ if let Some(char) = self.chars.pop_front() {
+ return Ok(Some(char));
+ }
+ if let Some(error) = self.error.take() {
+ return Err(error);
+ }
+ if self.eof {
+ return Ok(None);
+ }
+
+ self.read()?;
+
+ if let Some(char) = self.chars.pop_front() {
+ return Ok(Some(char));
+ }
+ if let Some(error) = self.error.take() {
+ return Err(error);
+ }
+ debug_assert!(self.eof);
+ Ok(None)
}
fn try_read_string(&mut self, s1: &str, case_sensitive: bool) -> Result<bool, Self::Error> {
debug_assert!(!s1.contains('\r'));
debug_assert!(!s1.contains('\n'));
+ debug_assert!(s1.len() <= self.buffer.len());
- if let Some(s2) = self.get_remaining_line()?.get(..s1.len()) {
- if s1 == s2 || (!case_sensitive && s1.eq_ignore_ascii_case(s2)) {
- self.line_pos += s1.len();
- return Ok(true);
+ while self.chars.len() < s1.len() {
+ if self.error.is_some() {
+ return Ok(false);
}
+ if self.eof {
+ return Ok(false);
+ }
+ self.read()?;
}
- Ok(false)
+ for (c, expected) in std::iter::zip(self.chars.iter(), s1.chars()) {
+ if case_sensitive {
+ if *c != expected {
+ return Ok(false);
+ }
+ } else {
+ if !c.eq_ignore_ascii_case(&expected) {
+ return Ok(false);
+ }
+ }
+ }
+
+ self.chars.drain(..s1.len());
+
+ Ok(true)
}
}
@@ -182,7 +266,6 @@ mod tests {
use super::{IntoReader, Reader};
#[test]
- #[should_panic] // FIXME
fn buf_read_reader_invalid_utf8() {
let mut reader = BufReader::new(b" \xc3\x28" as &[u8]).into_reader();
assert_eq!(reader.read_char().unwrap(), Some(' '));