report spans for errors

8 files changed, 61 insertions, 28 deletions

diff --git a/README.md b/README.md
index 95a0a05..0846542 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ changes:
   you had to do this yourself.
 
 * An optional `spans` feature has been added to make the tokenizer report the
-  source code spans for tag names, attribute names and attribute values.
+  source code spans for parser errors, tag names and attributes.
   The feature is disabled by default.
 
 * The API has been polished, e.g. the internal tokenizer state enums are no
diff --git a/examples/tokenize.rs b/examples/tokenize.rs
index 8728a18..1c9ea6a 100644
--- a/examples/tokenize.rs
+++ b/examples/tokenize.rs
@@ -66,9 +66,9 @@ impl TokenSink for TokenPrinter {
                 }
                 println!(">");
             }
-            ParseError(err) => {
+            ParseError { error, .. } => {
                 self.is_char(false);
-                println!("ERROR: {}", err);
+                println!("ERROR: {}", error);
             }
             _ => {
                 self.is_char(false);
diff --git a/src/tokenizer/error.rs b/src/tokenizer/error.rs
index 0acc88f..dad3fd2 100644
--- a/src/tokenizer/error.rs
+++ b/src/tokenizer/error.rs
@@ -6,7 +6,12 @@ use std::fmt::Display;
 pub enum Error {
     AttributesOnEndTag,
     SelfClosingEndTag,
-    DuplicateAttribute,
+    DuplicateAttribute {
+        #[cfg(feature = "spans")]
+        #[cfg_attr(docsrs, doc(cfg(feature = "spans")))]
+        /// Span of the duplicate attribute name.
+        span: std::ops::Range<usize>,
+    },
     BadCharacter(char),
     UnexpectedCharacter(char, InternalState),
     UnexpectedEOF(InternalState),
@@ -22,7 +27,7 @@ impl Display for Error {
         match self {
             Error::AttributesOnEndTag => write!(f, "attributes on an end tag"),
             Error::SelfClosingEndTag => write!(f, "self-closing end tag"),
-            Error::DuplicateAttribute => write!(f, "duplicate attribute"),
+            Error::DuplicateAttribute { .. } => write!(f, "duplicate attribute"),
             Error::BadCharacter(char) => write!(f, "bad character {:?}", char),
             Error::UnexpectedCharacter(char, state) => {
                 write!(
diff --git a/src/tokenizer/interface.rs b/src/tokenizer/interface.rs
index 715f9bc..128807e 100644
--- a/src/tokenizer/interface.rs
+++ b/src/tokenizer/interface.rs
@@ -112,7 +112,12 @@ pub enum Token {
     CharacterTokens(String),
     NullCharacterToken,
     EOFToken,
-    ParseError(Error),
+    ParseError {
+        error: Error,
+        #[cfg(feature = "spans")]
+        #[cfg_attr(docsrs, doc(cfg(feature = "spans")))]
+        span: std::ops::Range<usize>,
+    },
 }
 
 #[derive(Debug, PartialEq)]
diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs
index 6793eb2..1809275 100644
--- a/src/tokenizer/mod.rs
+++ b/src/tokenizer/mod.rs
@@ -538,7 +538,10 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
         };
 
         if dup {
-            self.emit_error(Error::DuplicateAttribute);
+            self.emit_error(Error::DuplicateAttribute {
+                #[cfg(feature = "spans")]
+                span: self.spans.current_attr_name.clone(),
+            });
             self.current_attr_name.clear();
             self.current_attr_value.clear();
         } else {
@@ -598,7 +601,11 @@ impl<Sink: TokenSink> Tokenizer<Sink> {
     }
 
     fn emit_error(&mut self, error: Error) {
-        self.process_token_and_continue(ParseError(error));
+        self.process_token_and_continue(ParseError {
+            error,
+            #[cfg(feature = "spans")]
+            span: self.spans.current_pos - 1..self.spans.current_pos - 1,
+        });
     }
 }
 //§ END
@@ -2293,7 +2300,7 @@ mod test {
                     self.current_str.push('\0');
                 }
 
-                token @ ParseError(_) => {
+                token @ ParseError { .. } => {
                     self.push(token, line_number);
                 }
 
@@ -2453,7 +2460,7 @@ mod test {
                 (3, CharacterTokens(c1)),
                 (
                     3,
-                    ParseError(Error::CharRef(CharRefError::InvalidNamedCharRef)),
+                    ParseError{error: Error::CharRef(CharRefError::InvalidNamedCharRef), ..},
                 ),
                 (4, CharacterTokens(c2)),
             ] if c1 == "&\n" && c2 == "&aamp;\n"
diff --git a/tests/files/test.html b/tests/files/test.html
index 0dcbdbf..14493b7 100644
--- a/tests/files/test.html
+++ b/tests/files/test.html
@@ -5,3 +5,7 @@ Here is a tag: <strong >very cool</strong>
 Tags can have attributes: <div id = foo >...</div>
 
 Attribute values can be quoted: <input name = 'age' type = "number">
+
+This is malformed < test
+
+Characters can be escaped but don't forget the semicolon: &#182
diff --git a/tests/files/test.out b/tests/files/test.out
index 7127ebc..f5acb3e 100644
--- a/tests/files/test.out
+++ b/tests/files/test.out
@@ -1,17 +1,23 @@
 note:
-  ┌─ test.html:3:17
-  │
-3 │ Here is a tag: <strong >very cool</strong>
-  │                 ^^^^^^             ^^^^^^ EndTag
-  │                 │
-  │                 StartTag
-4 │
-5 │ Tags can have attributes: <div id = foo >...</div>
-  │                                ^^   ^^^ attribute value
-  │                                │
-  │                                attribute name
-6 │
-7 │ Attribute values can be quoted: <input name = 'age' type = "number">
-  │                                                ^^^          ^^^^^^ in double quotes
-  │                                                │
-  │                                                in single quotes
+   ┌─ test.html:3:17
+   │
+ 3 │ Here is a tag: <strong >very cool</strong>
+   │                 ^^^^^^             ^^^^^^ EndTag
+   │                 │
+   │                 StartTag
+ 4 │
+ 5 │ Tags can have attributes: <div id = foo >...</div>
+   │                                ^^   ^^^ attribute value
+   │                                │
+   │                                attribute name
+ 6 │
+ 7 │ Attribute values can be quoted: <input name = 'age' type = "number">
+   │                                                ^^^          ^^^^^^ in double quotes
+   │                                                │
+   │                                                in single quotes
+ 8 │
+ 9 │ This is malformed < test
+   │                    ^ unexpected character: saw ' ' in state TagOpen
+10 │
+11 │ Characters can be escaped but don't forget the semicolon: &#182
+   │                                                               ^ semicolon missing after character reference
diff --git a/tests/spans.rs b/tests/spans.rs
index bfa42f6..5615853 100644
--- a/tests/spans.rs
+++ b/tests/spans.rs
@@ -1,5 +1,5 @@
 #![cfg(feature = "spans")]
-use std::include_str;
+use std::{include_str, ops::Range};
 
 use codespan_reporting::{
     self,
@@ -8,18 +8,21 @@ use codespan_reporting::{
     term::{self, termcolor::Buffer},
 };
 use html5tokenizer::{
-    BufferQueue, Tag, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
+    error::Error, BufferQueue, Tag, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
 };
 
 #[derive(Default)]
 struct TagSink {
     tags: Vec<Tag>,
+    errors: Vec<(Error, Range<usize>)>,
 }
 
 impl TokenSink for TagSink {
     fn process_token(&mut self, token: Token, _line_number: u64) -> TokenSinkResult {
         if let Token::TagToken(tag) = token {
             self.tags.push(tag);
+        } else if let Token::ParseError { error, span } = token {
+            self.errors.push((error, span));
         }
         TokenSinkResult::Continue
     }
@@ -61,6 +64,9 @@ fn test() {
         Label::primary(file_id, tags[4].attrs[1].value_span.clone())
             .with_message("in double quotes"),
     );
+    for (error, span) in tok.sink.errors {
+        labels.push(Label::primary(file_id, span).with_message(format!("{}", error)));
+    }
     let diagnostic = Diagnostic::note().with_labels(labels);
 
     let mut writer = Buffer::no_color();