use logos::Logos; use super::ModuleFunction; #[derive(Logos, Debug, PartialEq)] pub(crate) enum Token { #[token("{{")] OpenCall, #[token("\\}}")] EscapedCloseCall, #[token("}}")] CloseCall, #[regex("]*)?>")] OpenPre, #[regex("")] ClosePre, #[regex("")] OpenRaw, #[regex("")] CloseRaw, #[token("")] CloseComment, #[error] Other, } fn strip_marker(num: usize) -> String { format!("\x7fUNIQ-{:05X}-QINU\x7f", num) } #[derive(Logos, Debug, PartialEq)] pub(crate) enum MarkerToken { #[regex("\x7fUNIQ-[A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9]-QINU\x7f")] StripMarker, #[error] Other, } #[derive(Debug)] pub enum TokenType { OpenPre, ClosePre, OpenRaw, CloseRaw, OpenCall, CloseCall, } #[derive(Debug)] pub(crate) enum Sub<'a> { Call(&'a str), Raw(&'a str), Pre { attrs: &'a str, content: &'a str }, StrayToken(TokenType, &'a str), } pub(crate) struct ParserState<'a> { substitutions: Vec>, } impl<'a> ParserState<'a> { fn new() -> Self { Self { substitutions: Vec::new(), } } fn add(&mut self, sub: Sub<'a>) -> usize { let idx = self.substitutions.len(); self.substitutions.push(sub); idx } pub(crate) fn postprocess( &self, text: &str, render: impl Fn(&Sub<'a>, &mut String) -> std::fmt::Result, ) -> String { let mut out = String::new(); let mut lex = MarkerToken::lexer(text); while let Some(tok) = lex.next() { match tok { MarkerToken::StripMarker => match usize::from_str_radix(&lex.slice()[6..11], 16) { Ok(idx) => { render(&self.substitutions[idx], &mut out); } Err(e) => out.push_str(lex.slice()), }, MarkerToken::Other => out.push_str(lex.slice()), } } out.push_str(lex.remainder()); out } } pub(crate) fn preprocess(text: &str) -> (String, ParserState) { let mut stripped = String::new(); let mut lex = Token::lexer(text); let mut state = ParserState::new(); while let Some(tok) = lex.next() { match tok { Token::OpenComment => { let span = lex.span(); stripped.push_str(if lex.any(|t| matches!(&t, Token::CloseComment)) { &lex.source()[span.start..lex.span().end] } else { &lex.source()[span.start..] }); } Token::OpenRaw => { let span = lex.span(); let slice = lex.slice(); stripped.push_str(&strip_marker(state.add( if lex.any(|t| matches!(&t, Token::CloseRaw)) { Sub::Raw(&lex.source()[span.end..lex.span().start]) } else { lex = Token::lexer(&lex.source()[span.end..]); Sub::StrayToken(TokenType::OpenRaw, slice) }, ))); } Token::OpenCall => { let span = lex.span(); let slice = lex.slice(); stripped.push_str(&strip_marker(state.add( if lex.any(|t| matches!(&t, Token::CloseCall)) { Sub::Call(&lex.source()[span.end..lex.span().start]) } else { lex = Token::lexer(&lex.source()[span.end..]); Sub::StrayToken(TokenType::OpenCall, slice) }, ))); } Token::OpenPre => { let span = lex.span(); let slice = lex.slice(); stripped.push_str(&strip_marker(state.add( if lex.any(|t| matches!(&t, Token::ClosePre)) { Sub::Pre { attrs: &lex.source()[span.start + 4..span.end - 1], content: &lex.source()[span.end..lex.span().start], } } else { lex = Token::lexer(&lex.source()[span.end..]); Sub::StrayToken(TokenType::OpenPre, slice) }, ))); } Token::CloseCall => { stripped.push_str(&strip_marker( state.add(Sub::StrayToken(TokenType::CloseCall, lex.slice())), )); } Token::ClosePre => { stripped.push_str(&strip_marker( state.add(Sub::StrayToken(TokenType::ClosePre, lex.slice())), )); } Token::CloseRaw => { stripped.push_str(&strip_marker( state.add(Sub::StrayToken(TokenType::CloseRaw, lex.slice())), )); } _ => { stripped.push_str(lex.slice()); } } } stripped.push_str(lex.remainder()); (stripped, state) } #[cfg(test)] mod tests { use std::fmt::Write; use crate::lua::template::preprocess; use super::Sub; fn debug_format(sub: &Sub, out: &mut String) -> std::fmt::Result { write!(out, "{:?}", sub) } #[test] fn test_parse() { let (text, state) = preprocess("{{foobar}}
foo
{{test}}"); let out = state.postprocess(&text, debug_format); assert_eq!( out, r#"Call("foobar") Pre { attrs: " lang=js", content: "foo" } Raw("{{test")StrayToken(CloseCall, "}}")"# ); } #[test] fn test_stray_open_tokens() { let (text, state) = preprocess("{{
 ");
        let out = state.postprocess(&text, debug_format);
        assert_eq!(
            out,
            r#"StrayToken(OpenCall, "{{") StrayToken(OpenPre, "
") StrayToken(OpenRaw, "")"#
        );
    }

    #[test]
    fn test_stray_close_tokens() {
        let (text, state) = preprocess("}} 
"); let out = state.postprocess(&text, debug_format); assert_eq!( out, r#"StrayToken(CloseCall, "}}") StrayToken(ClosePre, "
") StrayToken(CloseRaw, "")"# ); } #[test] fn test_comment() { let (text, _state) = preprocess("