From 852d5c6f2e65a5ab466662ae1c649a0ed25c70a9 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Sun, 10 Sep 2023 19:37:34 +0200 Subject: break!: move offsets out of Token Previously the Token enum contained the offsets using the O generic type parameter, which could be a usize if you're tracking offsets or a zero-sized type if you didn't care about offsets. This commit moves all the byte offset and syntax information to a new Trace enum, which has several advantages: * Traces can now easily be stored separately, while the tokens are fed to the tree builder. (The tree builder only has to keep track of which tree nodes originate from which tokens.) * No needless generics for functions that take a token but don't care about offsets (a tree construction implementation is bound to have many of such functions). * The FromIterator<(String, String)> impl for AttributeMap no longer has to specify arbitrary values for the spans and the value_syntax). * The PartialEq implementation of Token is now much more useful (since it no longer includes all the offsets). * The Debug formatting of Token is now more readable (since it no longer includes all the offsets). * Function pointers to functions accepting tokens are possible. (Since function pointer types may not have generic parameters.) --- integration_tests/tests/test_html5lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'integration_tests') diff --git a/integration_tests/tests/test_html5lib.rs b/integration_tests/tests/test_html5lib.rs index eac11dd..42d93f1 100644 --- a/integration_tests/tests/test_html5lib.rs +++ b/integration_tests/tests/test_html5lib.rs @@ -107,7 +107,7 @@ fn run_test_inner( R: Reader + Position, O: Offset, E: Emitter + Iterator + DrainErrors, - T: Into>, + T: Into, { println!( "==== FILE {}, TEST {}, STATE {:?}, TOKENIZER {} ====", @@ -156,7 +156,7 @@ fn run_test_inner( actual_tokens.push(TestToken::Character(c.into())); } } - Token::Comment(comment) => actual_tokens.push(TestToken::Comment(comment.data)), + Token::Comment(comment) => actual_tokens.push(TestToken::Comment(comment)), Token::Doctype(doctype) => actual_tokens.push(TestToken::Doctype { name: doctype.name, public_id: doctype.public_id, -- cgit v1.2.3