use logos::Logos;
use super::ModuleFunction;
#[derive(Logos, Debug, PartialEq)]
pub(crate) enum Token {
#[token("{{")]
OpenCall,
#[token("\\}}")]
EscapedCloseCall,
#[token("}}")]
CloseCall,
#[regex("
]*)?>")]
OpenPre,
#[regex("
")]
ClosePre,
#[regex("")]
OpenRaw,
#[regex("")]
CloseRaw,
#[token("")]
CloseComment,
#[error]
Other,
}
fn strip_marker(num: usize) -> String {
format!("\x7fUNIQ-{:05X}-QINU\x7f", num)
}
#[derive(Logos, Debug, PartialEq)]
pub(crate) enum MarkerToken {
#[regex("\x7fUNIQ-[A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9]-QINU\x7f")]
StripMarker,
#[error]
Other,
}
#[derive(Debug)]
pub enum TokenType {
OpenPre,
ClosePre,
OpenRaw,
CloseRaw,
OpenCall,
CloseCall,
}
#[derive(Debug)]
pub(crate) enum Sub<'a> {
Call(&'a str),
Raw(&'a str),
Pre { attrs: &'a str, content: &'a str },
StrayToken(TokenType, &'a str),
}
pub(crate) struct ParserState<'a> {
substitutions: Vec>,
}
impl<'a> ParserState<'a> {
fn new() -> Self {
Self {
substitutions: Vec::new(),
}
}
fn add(&mut self, sub: Sub<'a>) -> usize {
let idx = self.substitutions.len();
self.substitutions.push(sub);
idx
}
pub(crate) fn postprocess(
&self,
text: &str,
render: impl Fn(&Sub<'a>, &mut String) -> std::fmt::Result,
) -> String {
let mut out = String::new();
let mut lex = MarkerToken::lexer(text);
while let Some(tok) = lex.next() {
match tok {
MarkerToken::StripMarker => match usize::from_str_radix(&lex.slice()[6..11], 16) {
Ok(idx) => {
let _ = render(&self.substitutions[idx], &mut out);
}
Err(_) => out.push_str(lex.slice()),
},
MarkerToken::Other => out.push_str(lex.slice()),
}
}
out.push_str(lex.remainder());
out
}
}
pub(crate) fn preprocess(text: &str) -> (String, ParserState) {
let mut stripped = String::new();
let mut lex = Token::lexer(text);
let mut state = ParserState::new();
while let Some(tok) = lex.next() {
match tok {
Token::OpenComment => {
let span = lex.span();
stripped.push_str(if lex.any(|t| matches!(&t, Token::CloseComment)) {
&lex.source()[span.start..lex.span().end]
} else {
&lex.source()[span.start..]
});
}
Token::OpenRaw => {
let span = lex.span();
let slice = lex.slice();
stripped.push_str(&strip_marker(state.add(
if lex.any(|t| matches!(&t, Token::CloseRaw)) {
Sub::Raw(&lex.source()[span.end..lex.span().start])
} else {
lex = Token::lexer(&lex.source()[span.end..]);
Sub::StrayToken(TokenType::OpenRaw, slice)
},
)));
}
Token::OpenCall => {
let span = lex.span();
let slice = lex.slice();
stripped.push_str(&strip_marker(state.add(
if lex.any(|t| matches!(&t, Token::CloseCall)) {
Sub::Call(&lex.source()[span.end..lex.span().start])
} else {
lex = Token::lexer(&lex.source()[span.end..]);
Sub::StrayToken(TokenType::OpenCall, slice)
},
)));
}
Token::OpenPre => {
let span = lex.span();
let slice = lex.slice();
stripped.push_str(&strip_marker(state.add(
if lex.any(|t| matches!(&t, Token::ClosePre)) {
Sub::Pre {
attrs: &lex.source()[span.start + 4..span.end - 1],
content: &lex.source()[span.end..lex.span().start],
}
} else {
lex = Token::lexer(&lex.source()[span.end..]);
Sub::StrayToken(TokenType::OpenPre, slice)
},
)));
}
Token::CloseCall => {
stripped.push_str(&strip_marker(
state.add(Sub::StrayToken(TokenType::CloseCall, lex.slice())),
));
}
Token::ClosePre => {
stripped.push_str(&strip_marker(
state.add(Sub::StrayToken(TokenType::ClosePre, lex.slice())),
));
}
Token::CloseRaw => {
stripped.push_str(&strip_marker(
state.add(Sub::StrayToken(TokenType::CloseRaw, lex.slice())),
));
}
_ => {
stripped.push_str(lex.slice());
}
}
}
stripped.push_str(lex.remainder());
(stripped, state)
}
#[cfg(test)]
mod tests {
use std::fmt::Write;
use crate::lua::template::preprocess;
use super::Sub;
fn debug_format(sub: &Sub, out: &mut String) -> std::fmt::Result {
write!(out, "{:?}", sub)
}
#[test]
fn test_parse() {
let (text, state) = preprocess("{{foobar}} foo
{{test}}");
let out = state.postprocess(&text, debug_format);
assert_eq!(
out,
r#"Call("foobar") Pre { attrs: " lang=js", content: "foo" } Raw("{{test")StrayToken(CloseCall, "}}")"#
);
}
#[test]
fn test_stray_open_tokens() {
let (text, state) = preprocess("{{ ");
let out = state.postprocess(&text, debug_format);
assert_eq!(
out,
r#"StrayToken(OpenCall, "{{") StrayToken(OpenPre, "") StrayToken(OpenRaw, "")"#
);
}
#[test]
fn test_stray_close_tokens() {
let (text, state) = preprocess("}}
");
let out = state.postprocess(&text, debug_format);
assert_eq!(
out,
r#"StrayToken(CloseCall, "}}") StrayToken(ClosePre, "
") StrayToken(CloseRaw, "")"#
);
}
#[test]
fn test_comment() {
let (text, _state) =
preprocess("