fix: BufReadReader skips line on invalid UTF-8

1 files changed, 109 insertions, 26 deletions

diff --git a/src/reader.rs b/src/reader.rs
index a987b28..15b9224 100644
--- a/src/reader.rs
+++ b/src/reader.rs
@@ -1,7 +1,8 @@
 //! Provides the [`Reader`] trait (and implementations) used by the tokenizer.
 
+use std::collections::VecDeque;
 use std::convert::Infallible;
-use std::io::{self, BufRead, BufReader, Read};
+use std::io::{self, BufReader, Read};
 
 /// An object that provides characters to the tokenizer.
 ///
@@ -27,6 +28,7 @@ pub trait Reader {
     ///
     /// If the next characters equal to `s`, this function consumes the respective characters from
     /// the input stream and returns `true`. If not, it does nothing and returns `false`.
+    // TODO: document a maximum s length that may be assumed (depends on longest named character reference ... which may change?)
     fn try_read_string(&mut self, s: &str, case_sensitive: bool) -> Result<bool, Self::Error>;
 }
 
@@ -112,57 +114,139 @@ impl<'a> IntoReader<'a> for &'a String {
     }
 }
 
-/// A [`BufRead`]-based [`Reader`] implementation that attempts to read UTF-8.
-pub struct BufReadReader<R: BufRead> {
-    line: String,
-    line_pos: usize,
+/// Just the same as [`std::sys_common::io::DEFAULT_BUF_SIZE`] (which isn't public).
+const BUF_SIZE: usize = 8 * 1024;
+
+/// A [`Read`]-based buffered [`Reader`] implementation that attempts to read UTF-8.
+pub struct BufReadReader<R: Read> {
     reader: R,
+    /// The buffer into which bytes will be read from the reader.
+    buffer: [u8; BUF_SIZE],
+    /// Number of bytes in the buffer that have been read.
+    read: usize,
+    /// Position in the buffer up until the bytes have been parsed to chars.
+    pos: usize,
+    /// The characters parsed from the buffer in read order.
+    chars: VecDeque<char>,
+    /// An error that has occurred after reading the current content of chars.
+    error: Option<io::Error>,
+    /// Indicates if the end-of-file has been reached (we won't read anymore).
+    eof: bool,
 }
 
-impl<R: BufRead> BufReadReader<R> {
-    /// Construct a new `BufReadReader` from any type that implements `BufRead`.
+impl<R: Read> BufReadReader<R> {
+    /// Construct a new `BufReadReader` from any type that implements [`Read`].
     pub fn new(reader: R) -> Self {
         BufReadReader {
-            line: String::new(),
-            line_pos: 0,
             reader,
+            buffer: [0; BUF_SIZE],
+            read: 0,
+            pos: 0,
+            chars: VecDeque::new(),
+            error: None,
+            eof: false,
         }
     }
 
     #[inline]
-    fn get_remaining_line(&mut self) -> Result<&str, io::Error> {
-        if self.line_pos < self.line.len() {
-            return Ok(&self.line[self.line_pos..]);
+    fn read(&mut self) -> Result<(), io::Error> {
+        debug_assert!(!self.eof);
+        debug_assert!(self.error.is_none());
+
+        if self.pos == self.read {
+            self.read = match self.reader.read(&mut self.buffer)? {
+                0 => {
+                    self.eof = true;
+                    return Ok(());
+                }
+                n => n,
+            };
+            self.pos = 0;
+        }
+
+        let unprocessed = &self.buffer[self.pos..self.read];
+
+        let (valid_str, err) = match std::str::from_utf8(unprocessed) {
+            Ok(s) => (s, None),
+            Err(err) => (
+                // SAFETY: The UTF-8 checking has already been done by the previous from_utf8 call.
+                unsafe { std::str::from_utf8_unchecked(&unprocessed[..err.valid_up_to()]) },
+                Some(err),
+            ),
+        };
+        for c in valid_str.chars() {
+            self.chars.push_back(c);
         }
+        self.pos += valid_str.len();
 
-        self.line.clear();
-        self.line_pos = 0;
-        self.reader.read_line(&mut self.line)?;
-        Ok(&self.line)
+        if let Some(err) = err {
+            self.error = Some(io::Error::new(io::ErrorKind::InvalidData, err));
+
+            match err.error_len() {
+                None => self.eof = true,
+                Some(error_len) => self.pos += error_len,
+            }
+        }
+        Ok(())
     }
 }
 
-impl<R: BufRead> Reader for BufReadReader<R> {
+impl<R: Read> Reader for BufReadReader<R> {
     type Error = io::Error;
 
     fn read_char(&mut self) -> Result<Option<char>, Self::Error> {
-        let rv = self.get_remaining_line()?.chars().next();
-        self.line_pos += rv.map(char::len_utf8).unwrap_or(1);
-        Ok(rv)
+        if let Some(char) = self.chars.pop_front() {
+            return Ok(Some(char));
+        }
+        if let Some(error) = self.error.take() {
+            return Err(error);
+        }
+        if self.eof {
+            return Ok(None);
+        }
+
+        self.read()?;
+
+        if let Some(char) = self.chars.pop_front() {
+            return Ok(Some(char));
+        }
+        if let Some(error) = self.error.take() {
+            return Err(error);
+        }
+        debug_assert!(self.eof);
+        Ok(None)
     }
 
     fn try_read_string(&mut self, s1: &str, case_sensitive: bool) -> Result<bool, Self::Error> {
         debug_assert!(!s1.contains('\r'));
         debug_assert!(!s1.contains('\n'));
+        debug_assert!(s1.len() <= self.buffer.len());
 
-        if let Some(s2) = self.get_remaining_line()?.get(..s1.len()) {
-            if s1 == s2 || (!case_sensitive && s1.eq_ignore_ascii_case(s2)) {
-                self.line_pos += s1.len();
-                return Ok(true);
+        while self.chars.len() < s1.len() {
+            if self.error.is_some() {
+                return Ok(false);
             }
+            if self.eof {
+                return Ok(false);
+            }
+            self.read()?;
         }
 
-        Ok(false)
+        for (c, expected) in std::iter::zip(self.chars.iter(), s1.chars()) {
+            if case_sensitive {
+                if *c != expected {
+                    return Ok(false);
+                }
+            } else {
+                if !c.eq_ignore_ascii_case(&expected) {
+                    return Ok(false);
+                }
+            }
+        }
+
+        self.chars.drain(..s1.len());
+
+        Ok(true)
     }
 }
 
@@ -182,7 +266,6 @@ mod tests {
     use super::{IntoReader, Reader};
 
     #[test]
-    #[should_panic] // FIXME
     fn buf_read_reader_invalid_utf8() {
         let mut reader = BufReader::new(b" \xc3\x28" as &[u8]).into_reader();
         assert_eq!(reader.read_char().unwrap(), Some(' '));