diff options
| -rw-r--r-- | Cargo.lock | 36 | ||||
| -rw-r--r-- | Cargo.toml | 3 | ||||
| -rw-r--r-- | README.md | 33 | ||||
| -rw-r--r-- | src/get_routes.rs | 2 | ||||
| -rw-r--r-- | src/lua.rs | 44 | ||||
| -rw-r--r-- | src/lua/template.rs | 333 | ||||
| -rw-r--r-- | src/main.rs | 49 | ||||
| -rw-r--r-- | src/post_routes.rs | 2 |
8 files changed, 493 insertions, 9 deletions
@@ -20,6 +20,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + +[[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -226,6 +232,7 @@ dependencies = [ "httpdate", "hyper", "hyperlocal", + "logos", "mime_guess", "multer", "percent-encoding", @@ -412,6 +419,29 @@ dependencies = [ ] [[package]] +name = "logos" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8b031682c67a8e3d5446840f9573eb7fe26efe7ec8d195c9ac4c0647c502f1" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d849148dbaf9661a6151d1ca82b13bb4c4c128146a88d05253b38d4e2f496c" +dependencies = [ + "beef", + "fnv", + "proc-macro2", + "quote", + "regex-syntax", + "syn", +] + +[[package]] name = "memchr" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -640,6 +670,12 @@ dependencies = [ ] [[package]] +name = "regex-syntax" +version = "0.6.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" + +[[package]] name = "remove_dir_all" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -13,7 +13,7 @@ categories = ["command-line-utilities"] [features] default = ["lua"] -lua = ["rlua"] +lua = ["rlua", "logos"] [dependencies] hyper = { version = "0.14.20", features=["http1", "server", "runtime", "stream"]} @@ -39,6 +39,7 @@ multer = "2.0.3" mime_guess = "2.0" camino = "1" rlua = { version = "0.19.4", optional = true } +logos = { version = "0.12.1", optional = true } [target.'cfg(unix)'.dependencies] hyperlocal = { version = "0.8", features = ["server"] } @@ -80,7 +80,38 @@ name = "John Doe" email = "john@example.com" ``` -## Lua scripting +## Lua templates + +You can define Lua functions in `.lua` files within the `modules/` directory +and then call them within `.md` files using the `{{modulename.functionname}}` +syntax. For example if you create `modules/example.lua` with: + +```lua +function greet(args) + return 'Hello ' .. args[1] .. '!' +end + +function sum(args) + return args.x + args.y +end + +return {greet=greet, sum=sum} +``` + +within Markdown files `{{example.greet|world}}` will become +`Hello world!` and `{{example.sum|x=3|y=5}}` will become 8. + +Note that: + +* If no module name is given it defaults to `default` so e.g. `{{foo}}` + will attempt to call the `foo` function defined in `modules/default.lua`. +* In order to pass `|` or `}}` within arguments you need to + escape them by prefixing a slash e.g. `{{foo| \| \}} }}`. +* Within `<pre>` and `<raw>` tags and HTML comments no special syntax is interpreted. +* Note that none of these constructs can be nested. E.g. `{{foo|{{bar}} }}` or + `{{foo|<raw>test</raw>}}` will not work as you might expect. + +## Lua shebangs Files can start with a shebang like `#!hello`, which will interpret the following text with the `view` function returned diff --git a/src/get_routes.rs b/src/get_routes.rs index 5e65637..951c7b6 100644 --- a/src/get_routes.rs +++ b/src/get_routes.rs @@ -109,7 +109,7 @@ fn view_blob<C: Controller>( return Ok(page.into()); } if let Some(renderer) = get_renderer(&ctx.path) { - renderer(text, &mut page, RenderMode::View); + renderer(text, &mut page, RenderMode::View, &ctx); } else { page.body .push_str(&format!("<pre>{}</pre>", html_escape(text))); @@ -11,7 +11,10 @@ use rlua::Table; use crate::Context; +use self::template::ArgIndex; + mod serde; +pub mod template; pub struct Script<'a> { pub lua_module_name: &'a str, @@ -113,6 +116,47 @@ impl<'a> Script<'a> { } } +fn module_path(module_name: &str) -> String { + format!("modules/{}.lua", module_name) +} + +pub struct ModuleFunction<'a> { + pub module_name: &'a str, + pub function_name: &'a str, +} + +pub fn call( + modfn: &ModuleFunction, + args: impl Iterator<Item = (ArgIndex, String)>, + ctx: &Context, +) -> Result<String, ScriptError> { + let filename = module_path(modfn.module_name); + + let lua_entr = ctx + .branch_head() + .unwrap() + .tree() + .and_then(|tree| tree.get_path(Path::new(&filename))) + .map_err(|_| ScriptError::ModuleNotFound)?; + + let lua_blob = ctx.repo.find_blob(lua_entr.id()).unwrap(); + let lua_code = from_utf8(lua_blob.content()).map_err(|_| ScriptError::ModuleNotUtf8)?; + + lua_context(|ctx| { + let module: Table = ctx.load(lua_code).eval()?; + let view: Function = module.get(modfn.function_name)?; + let lua_args = ctx.create_table()?; + for (idx, val) in args { + match idx { + ArgIndex::Str(s) => lua_args.set(s, val)?, + ArgIndex::Num(n) => lua_args.set(n, val)?, + } + } + view.call(lua_args) + }) + .map_err(ScriptError::LuaError) +} + impl Display for ScriptError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/src/lua/template.rs b/src/lua/template.rs new file mode 100644 index 0000000..de13481 --- /dev/null +++ b/src/lua/template.rs @@ -0,0 +1,333 @@ +use logos::Logos; + +use super::ModuleFunction; + +#[derive(Logos, Debug, PartialEq)] +pub(crate) enum Token { + #[token("{{")] + OpenCall, + + #[token("\\}}")] + EscapedCloseCall, + + #[token("}}")] + CloseCall, + + #[regex("<pre( +[^>]*)?>")] + OpenPre, + + #[regex("</pre>")] + ClosePre, + + #[regex("<raw>")] + OpenRaw, + + #[regex("</raw>")] + CloseRaw, + + #[token("<!--")] + OpenComment, + + #[token("-->")] + CloseComment, + + #[error] + Other, +} + +fn strip_marker(num: usize) -> String { + format!("\x7fUNIQ-{:05X}-QINU\x7f", num) +} + +#[derive(Logos, Debug, PartialEq)] +pub(crate) enum MarkerToken { + #[regex("\x7fUNIQ-[A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9]-QINU\x7f")] + StripMarker, + + #[error] + Other, +} + +#[derive(Debug)] +pub enum TokenType { + OpenPre, + ClosePre, + OpenRaw, + CloseRaw, + OpenCall, + CloseCall, +} + +#[derive(Debug)] +pub(crate) enum Sub<'a> { + Call(&'a str), + Raw(&'a str), + Pre { attrs: &'a str, content: &'a str }, + StrayToken(TokenType, &'a str), +} + +pub(crate) struct ParserState<'a> { + substitutions: Vec<Sub<'a>>, +} + +impl<'a> ParserState<'a> { + fn new() -> Self { + Self { + substitutions: Vec::new(), + } + } + + fn add(&mut self, sub: Sub<'a>) -> usize { + let idx = self.substitutions.len(); + self.substitutions.push(sub); + idx + } + + pub(crate) fn postprocess( + &self, + text: &str, + render: impl Fn(&Sub<'a>, &mut String) -> std::fmt::Result, + ) -> String { + let mut out = String::new(); + let mut lex = MarkerToken::lexer(text); + while let Some(tok) = lex.next() { + match tok { + MarkerToken::StripMarker => match usize::from_str_radix(&lex.slice()[6..11], 16) { + Ok(idx) => { + render(&self.substitutions[idx], &mut out); + } + Err(e) => out.push_str(lex.slice()), + }, + MarkerToken::Other => out.push_str(lex.slice()), + } + } + out.push_str(lex.remainder()); + out + } +} + +pub(crate) fn preprocess(text: &str) -> (String, ParserState) { + let mut stripped = String::new(); + let mut lex = Token::lexer(text); + let mut state = ParserState::new(); + + while let Some(tok) = lex.next() { + match tok { + Token::OpenComment => { + let span = lex.span(); + + stripped.push_str(if lex.any(|t| matches!(&t, Token::CloseComment)) { + &lex.source()[span.start..lex.span().end] + } else { + &lex.source()[span.start..] + }); + } + Token::OpenRaw => { + let span = lex.span(); + let slice = lex.slice(); + + stripped.push_str(&strip_marker(state.add( + if lex.any(|t| matches!(&t, Token::CloseRaw)) { + Sub::Raw(&lex.source()[span.end..lex.span().start]) + } else { + lex = Token::lexer(&lex.source()[span.end..]); + Sub::StrayToken(TokenType::OpenRaw, slice) + }, + ))); + } + Token::OpenCall => { + let span = lex.span(); + let slice = lex.slice(); + + stripped.push_str(&strip_marker(state.add( + if lex.any(|t| matches!(&t, Token::CloseCall)) { + Sub::Call(&lex.source()[span.end..lex.span().start]) + } else { + lex = Token::lexer(&lex.source()[span.end..]); + Sub::StrayToken(TokenType::OpenCall, slice) + }, + ))); + } + Token::OpenPre => { + let span = lex.span(); + let slice = lex.slice(); + + stripped.push_str(&strip_marker(state.add( + if lex.any(|t| matches!(&t, Token::ClosePre)) { + Sub::Pre { + attrs: &lex.source()[span.start + 4..span.end - 1], + content: &lex.source()[span.end..lex.span().start], + } + } else { + lex = Token::lexer(&lex.source()[span.end..]); + Sub::StrayToken(TokenType::OpenPre, slice) + }, + ))); + } + Token::CloseCall => { + stripped.push_str(&strip_marker( + state.add(Sub::StrayToken(TokenType::CloseCall, lex.slice())), + )); + } + Token::ClosePre => { + stripped.push_str(&strip_marker( + state.add(Sub::StrayToken(TokenType::ClosePre, lex.slice())), + )); + } + Token::CloseRaw => { + stripped.push_str(&strip_marker( + state.add(Sub::StrayToken(TokenType::CloseRaw, lex.slice())), + )); + } + _ => { + stripped.push_str(lex.slice()); + } + } + } + stripped.push_str(lex.remainder()); + + (stripped, state) +} + +#[cfg(test)] +mod tests { + use std::fmt::Write; + + use crate::lua::template::preprocess; + + use super::Sub; + + fn debug_format(sub: &Sub, out: &mut String) -> std::fmt::Result { + write!(out, "{:?}", sub) + } + + #[test] + fn test_parse() { + let (text, state) = preprocess("{{foobar}} <pre lang=js>foo</pre> <raw>{{test</raw>}}"); + let out = state.postprocess(&text, debug_format); + assert_eq!( + out, + r#"Call("foobar") Pre { attrs: " lang=js", content: "foo" } Raw("{{test")StrayToken(CloseCall, "}}")"# + ); + } + + #[test] + fn test_stray_open_tokens() { + let (text, state) = preprocess("{{ <pre> <raw>"); + let out = state.postprocess(&text, debug_format); + assert_eq!( + out, + r#"StrayToken(OpenCall, "{{") StrayToken(OpenPre, "<pre>") StrayToken(OpenRaw, "<raw>")"# + ); + } + + #[test] + fn test_stray_close_tokens() { + let (text, state) = preprocess("}} </pre> </raw>"); + let out = state.postprocess(&text, debug_format); + assert_eq!( + out, + r#"StrayToken(CloseCall, "}}") StrayToken(ClosePre, "</pre>") StrayToken(CloseRaw, "</raw>")"# + ); + } + + #[test] + fn test_comment() { + let (text, _state) = + preprocess("<!-- {{foobar}} <pre lang=js>foo</pre> <raw>{{test</raw>}}"); + assert_eq!( + text, + r#"<!-- {{foobar}} <pre lang=js>foo</pre> <raw>{{test</raw>}}"# + ); + } + + #[test] + fn test_call_after_stray_open() { + let (text, state) = preprocess("<pre> {{foo}}"); + let out = state.postprocess(&text, debug_format); + assert_eq!(out, r#"StrayToken(OpenPre, "<pre>") Call("foo")"#); + } +} + +#[derive(Logos, Debug, PartialEq)] +enum CallInnerToken { + #[token("\\|")] + EscapedPipe, + + #[regex("\\}}")] + EscapedClose, + + #[token("|")] + Pipe, + + #[error] + Other, +} + +#[derive(Debug)] +pub enum ArgIndex { + Str(String), + Num(usize), +} + +fn parse_arg(text: &str, cur_idx: &mut usize) -> (ArgIndex, String) { + if let Some((key, val)) = text.split_once('=') { + let key = key.trim(); + let idx = match key.parse::<usize>() { + Ok(n) => ArgIndex::Num(n), + Err(_) => ArgIndex::Str(key.to_string()), + }; + (idx, val.trim().to_string()) + } else { + let ret = (ArgIndex::Num(*cur_idx), text.trim().to_string()); + *cur_idx += 1; + ret + } +} + +pub fn parse_call_args(text: &str) -> (ModuleFunction, std::vec::IntoIter<(ArgIndex, String)>) { + let mut args = Vec::new(); + let mut arg = String::new(); + + let mut iter = text.splitn(2, '|'); + let arg0 = iter.next().unwrap(); + + let mut idx = 1; + + if let Some(rest) = iter.next() { + let mut lexer = CallInnerToken::lexer(rest); + while let Some(tok) = lexer.next() { + match tok { + CallInnerToken::EscapedClose => arg.push_str("}}"), + CallInnerToken::EscapedPipe => arg.push('|'), + CallInnerToken::Other => arg.push_str(lexer.slice()), + CallInnerToken::Pipe => { + args.push(parse_arg(&arg, &mut idx)); + arg = String::new(); + idx += 1; + } + } + } + arg.push_str(lexer.remainder()); + args.push(parse_arg(&arg, &mut idx)); + } + + let mut parts = arg0.splitn(2, '.'); + + let module_name; + let mut function_name = parts.next().unwrap(); + if let Some(next) = parts.next() { + module_name = function_name; + function_name = next; + } else { + module_name = "default"; + } + + ( + ModuleFunction { + module_name, + function_name, + }, + args.into_iter(), + ) +} diff --git a/src/main.rs b/src/main.rs index 2cd3a5d..e2fea85 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,6 @@ use hyper::StatusCode; use hyper::{Body, Server}; use origins::HttpOrigin; use percent_encoding::percent_decode_str; -use pulldown_cmark::html; use pulldown_cmark::Options; use pulldown_cmark::Parser; use serde::Deserialize; @@ -28,6 +27,7 @@ use sputnik::request::SputnikParts; use sputnik::response::SputnikBuilder; use std::convert::Infallible; use std::env; +use std::fmt::Write; use std::path::Path; use std::sync::Arc; @@ -516,14 +516,53 @@ enum RenderMode { Preview, } -fn render_markdown(input: &str, page: &mut Page, _mode: RenderMode) { - let parser = Parser::new_ext(input, Options::all()); +fn render_markdown(text: &str, page: &mut Page, _mode: RenderMode, ctx: &Context) { + let md_options = Options::all(); + page.body.push_str("<div class=markdown-output>"); - html::push_html(&mut page.body, parser); + #[cfg(feature = "lua")] + { + use lua::template::{parse_call_args, Sub}; + + let (text, state) = lua::template::preprocess(text); + let parser = Parser::new_ext(&text, md_options); + let mut out = String::new(); + pulldown_cmark::html::push_html(&mut out, parser); + + let out = state.postprocess(&out, |sub, out| match *sub { + Sub::Raw(raw) => out.write_str(&html_escape(raw)), + Sub::Pre { attrs, content } => { + write!(out, "<pre {attrs}>{}</pre>", html_escape(content)) + } + Sub::StrayToken(_, text) => out.write_str(&html_escape(text)), + Sub::Call(text) => { + let (modfn, args) = parse_call_args(text); + + // TODO: cache module source code to avoid unnecessary git reads for repeated templates + match lua::call(&modfn, args, ctx) { + Ok(output) => out.write_str(&output), + Err(err) => write!( + out, + "<span class=error>{}: {}</span>", + html_escape(modfn.module_name), // TODO: link module + &html_escape(err.to_string()), + ), + } + } + }); + page.body.push_str(&out); + } + #[cfg(not(feature = "lua"))] + { + let parser = Parser::new_ext(text, md_options); + let mut out = String::new(); + pulldown_cmark::html::push_html(&mut out, parser); + page.body.push_str(&out); + } page.body.push_str("</div>"); } -fn get_renderer(path: &Utf8Path) -> Option<fn(&str, &mut Page, RenderMode)> { +fn get_renderer(path: &Utf8Path) -> Option<fn(&str, &mut Page, RenderMode, ctx: &Context)> { match path.extension() { Some("md") => Some(render_markdown), _ => None, diff --git a/src/post_routes.rs b/src/post_routes.rs index 51b35ec..fda1619 100644 --- a/src/post_routes.rs +++ b/src/post_routes.rs @@ -361,6 +361,6 @@ async fn preview_edit<C: Controller>( let form: EditForm = body.into_form().await?; let new_text = form.text.replace("\r\n", "\n"); let mut page = edit_text_form(&form, None, controller, &ctx, parts); - get_renderer(&ctx.path).unwrap()(&new_text, &mut page, RenderMode::Preview); + get_renderer(&ctx.path).unwrap()(&new_text, &mut page, RenderMode::Preview, &ctx); Ok(page.into()) } |