From bb7c64a8245769f20742502cfa942f33ccb08af7 Mon Sep 17 00:00:00 2001
From: Martin Fischer <martin@push-f.com>
Date: Tue, 29 Aug 2023 13:17:59 +0200
Subject: break!: add Token::EndOfFile

While the end-of-file token can also be represented by None,
this is less clear than having an explicit variant. Especially when
it comes to tree construction, the spec explicitly has conditions
named "An end-of-file token", and it's nice if the code for tree
construction can match the spec text closely.
---
 CHANGELOG.md                             | 2 ++
 integration_tests/tests/test_html5lib.rs | 1 +
 src/token.rs                             | 2 ++
 3 files changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 075373c..e59e008 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,8 @@
   * Replaced the `String` variant with a new `Char` variant.  
     (The tokenizer now emits chars instead of strings.)
 
+  * Added the `EndOfFile` variant.
+
 * `Emitter` trait
 
   * Removed `pop_token` method and `Token` associated type.
diff --git a/integration_tests/tests/test_html5lib.rs b/integration_tests/tests/test_html5lib.rs
index bede29a..8fedc1a 100644
--- a/integration_tests/tests/test_html5lib.rs
+++ b/integration_tests/tests/test_html5lib.rs
@@ -152,6 +152,7 @@ fn run_test_inner<R: Reader>(
                 system_id: doctype.system_id,
                 force_quirks: doctype.force_quirks,
             }),
+            Token::EndOfFile => {}
         };
     }
 
diff --git a/src/token.rs b/src/token.rs
index cb584ff..3104a61 100644
--- a/src/token.rs
+++ b/src/token.rs
@@ -21,6 +21,8 @@ pub enum Token<O> {
     Comment(Comment<O>),
     /// An HTML doctype declaration.
     Doctype(Doctype<O>),
+    /// An end-of-file token.
+    EndOfFile,
 }
 
 /// An HTML start tag, such as `<p>` or `<a>`.
-- 
cgit v1.2.3