From df5a2ae14c4976d404195e83e52b8db62be63988 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Tue, 29 Aug 2023 07:24:55 +0200 Subject: break!: emit chars instead of strings The HTML spec specifies that the tokenizer emits character tokens. That html5gum always emitted strings instead was probably just done to make the token consumption more convenient. When it comes to tree construction character tokens are however actually more convenient than string tokens since the spec defines that specific character tokens should be ignored in specific states (and character tokens let us avoid string manipulation for these conditions). This should also make the DefaultEmitter more performant for cases where you don't actually need the strings at all (or only a few) since it avoids string allocations. Though I haven't benchmarked it. --- integration_tests/tests/test_html5lib.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'integration_tests') diff --git a/integration_tests/tests/test_html5lib.rs b/integration_tests/tests/test_html5lib.rs index 2e404c5..bede29a 100644 --- a/integration_tests/tests/test_html5lib.rs +++ b/integration_tests/tests/test_html5lib.rs @@ -137,7 +137,14 @@ fn run_test_inner( self_closing: tag.self_closing, }), Token::EndTag(tag) => actual_tokens.push(TestToken::EndTag { name: tag.name }), - Token::String(data) => actual_tokens.push(TestToken::Character(data)), + Token::Char(c) => { + // Coalesce all adjacent character tokens into a single string. + if let Some(TestToken::Character(s)) = actual_tokens.last_mut() { + s.push(c); + } else { + actual_tokens.push(TestToken::Character(c.into())); + } + } Token::Comment(comment) => actual_tokens.push(TestToken::Comment(comment.data)), Token::Doctype(doctype) => actual_tokens.push(TestToken::Doctype { name: doctype.name, -- cgit v1.2.3