make test sink also emit CharacterTokens & ParseError

1 files changed, 19 insertions, 5 deletions

diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs
index bcbc6b7..0acdcaf 100644
--- a/src/tokenizer/mod.rs
+++ b/src/tokenizer/mod.rs
@@ -1525,16 +1525,16 @@ mod test {
     // if current_line is being updated when process_token is called. The lines
     // vector is a collection of the line numbers that each token is on.
     struct LinesMatch {
-        tokens: Vec<Token>,
         current_str: String,
+        current_str_line: u64,
         lines: Vec<(Token, u64)>,
     }
 
     impl LinesMatch {
         fn new() -> LinesMatch {
             LinesMatch {
-                tokens: vec![],
                 current_str: String::new(),
+                current_str_line: 0,
                 lines: vec![],
             }
         }
@@ -1547,7 +1547,8 @@ mod test {
         fn finish_str(&mut self) {
             if self.current_str.len() > 0 {
                 let s = replace(&mut self.current_str, String::new());
-                self.tokens.push(CharacterTokens(s));
+                self.push(CharacterTokens(s), self.current_str_line);
+                self.current_str_line = 0;
             }
         }
     }
@@ -1555,6 +1556,10 @@ mod test {
     impl TokenSink for LinesMatch {
         type Handle = ();
 
+        fn end(&mut self) {
+            self.finish_str();
+        }
+
         fn process_token(
             &mut self,
             token: Token,
@@ -1562,6 +1567,7 @@ mod test {
         ) -> TokenSinkResult<Self::Handle> {
             match token {
                 CharacterTokens(b) => {
+                    self.current_str_line = line_number;
                     self.current_str.push_str(&b);
                 }
 
@@ -1569,8 +1575,8 @@ mod test {
                     self.current_str.push('\0');
                 }
 
-                ParseError(_) => {
-                    panic!("unexpected parse error");
+                token @ ParseError(_) => {
+                    self.push(token, line_number);
                 }
 
                 TagToken(mut t) => {
@@ -1659,9 +1665,13 @@ mod test {
         ];
         let expected = vec![
             (create_tag(String::from("a"), StartTag), 1),
+            (CharacterTokens("\n".into()), 2),
             (create_tag(String::from("b"), StartTag), 2),
+            (CharacterTokens("\n".into()), 3),
             (create_tag(String::from("b"), EndTag), 3),
+            (CharacterTokens("\n".into()), 4),
             (create_tag(String::from("a"), EndTag), 4),
+            (CharacterTokens("\n".into()), 5),
         ];
         let results = tokenize(vector, opts);
         assert_eq!(results, expected);
@@ -1684,9 +1694,13 @@ mod test {
         ];
         let expected = vec![
             (create_tag(String::from("a"), StartTag), 1),
+            (CharacterTokens("\n".into()), 2),
             (create_tag(String::from("b"), StartTag), 2),
+            (CharacterTokens("\n".into()), 3),
             (create_tag(String::from("b"), EndTag), 3),
+            (CharacterTokens("\n".into()), 4),
             (create_tag(String::from("a"), EndTag), 4),
+            (CharacterTokens("\n".into()), 5),
         ];
         let results = tokenize(vector, opts);
         assert_eq!(results, expected);