1 files changed, 12 insertions, 10 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 2580144..96d1c34 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -313,18 +313,20 @@ where
         loop {
             if let Some(token) = self.emitter.pop_token() {
                 return Some(Ok(token));
-            } else if !self.eof {
-                match machine::consume(self) {
-                    Ok(ControlToken::Continue) => (),
-                    Ok(ControlToken::Eof) => {
-                        self.eof = true;
-                        self.emitter.emit_eof();
-                    }
-                    Err(e) => return Some(Err(e)),
-                }
-            } else {
+            }
+
+            if self.eof {
                 return None;
             }
+
+            match machine::consume(self) {
+                Err(e) => return Some(Err(e)),
+                Ok(ControlToken::Continue) => (),
+                Ok(ControlToken::Eof) => {
+                    self.eof = true;
+                    self.emitter.emit_eof();
+                }
+            }
         }
     }
 }