implement lua-based templates

8 files changed, 493 insertions, 9 deletions

diff --git a/Cargo.lock b/Cargo.lock
index c1d2283..c53e499 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -20,6 +20,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
+name = "beef"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
+
+[[package]]
 name = "bitflags"
 version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -226,6 +232,7 @@ dependencies = [
  "httpdate",
  "hyper",
  "hyperlocal",
+ "logos",
  "mime_guess",
  "multer",
  "percent-encoding",
@@ -412,6 +419,29 @@ dependencies = [
 ]
 
 [[package]]
+name = "logos"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf8b031682c67a8e3d5446840f9573eb7fe26efe7ec8d195c9ac4c0647c502f1"
+dependencies = [
+ "logos-derive",
+]
+
+[[package]]
+name = "logos-derive"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d849148dbaf9661a6151d1ca82b13bb4c4c128146a88d05253b38d4e2f496c"
+dependencies = [
+ "beef",
+ "fnv",
+ "proc-macro2",
+ "quote",
+ "regex-syntax",
+ "syn",
+]
+
+[[package]]
 name = "memchr"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -640,6 +670,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "regex-syntax"
+version = "0.6.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
+
+[[package]]
 name = "remove_dir_all"
 version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 6e22549..4e7474e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,7 +13,7 @@ categories = ["command-line-utilities"]
 
 [features]
 default = ["lua"]
-lua = ["rlua"]
+lua = ["rlua", "logos"]
 
 [dependencies]
 hyper = { version = "0.14.20",  features=["http1", "server", "runtime", "stream"]}
@@ -39,6 +39,7 @@ multer = "2.0.3"
 mime_guess = "2.0"
 camino = "1"
 rlua = { version = "0.19.4", optional = true }
+logos = { version = "0.12.1", optional = true }
 
 [target.'cfg(unix)'.dependencies]
 hyperlocal = { version = "0.8", features = ["server"] }
diff --git a/README.md b/README.md
index 068c1fa..9a2ba59 100644
--- a/README.md
+++ b/README.md
@@ -80,7 +80,38 @@ name = "John Doe"
 email = "john@example.com"
 ```
 
-## Lua scripting
+## Lua templates
+
+You can define Lua functions in `.lua` files within the `modules/` directory
+and then call them within `.md` files using the `{{modulename.functionname}}`
+syntax. For example if you create `modules/example.lua` with:
+
+```lua
+function greet(args)
+    return 'Hello ' .. args[1] .. '!'
+end
+
+function sum(args)
+    return args.x + args.y
+end
+
+return {greet=greet, sum=sum}
+```
+
+within Markdown files `{{example.greet|world}}` will become
+`Hello world!` and `{{example.sum|x=3|y=5}}` will become 8.
+
+Note that:
+
+* If no module name is given it defaults to `default` so e.g. `{{foo}}`
+  will attempt to call the `foo` function defined in `modules/default.lua`.
+* In order to pass `|` or `}}` within arguments you need to
+  escape them by prefixing a slash e.g. `{{foo| \| \}} }}`.
+* Within `<pre>` and `<raw>` tags and HTML comments no special syntax is interpreted.
+* Note that none of these constructs can be nested. E.g. `{{foo|{{bar}} }}` or
+  `{{foo|<raw>test</raw>}}` will not work as you might expect.
+
+## Lua shebangs
 
 Files can start with a shebang like `#!hello`, which will
 interpret the following text with the `view` function returned
diff --git a/src/get_routes.rs b/src/get_routes.rs
index 5e65637..951c7b6 100644
--- a/src/get_routes.rs
+++ b/src/get_routes.rs
@@ -109,7 +109,7 @@ fn view_blob<C: Controller>(
                 return Ok(page.into());
             }
             if let Some(renderer) = get_renderer(&ctx.path) {
-                renderer(text, &mut page, RenderMode::View);
+                renderer(text, &mut page, RenderMode::View, &ctx);
             } else {
                 page.body
                     .push_str(&format!("<pre>{}</pre>", html_escape(text)));
diff --git a/src/lua.rs b/src/lua.rs
index 2ab5624..5caf3f1 100644
--- a/src/lua.rs
+++ b/src/lua.rs
@@ -11,7 +11,10 @@ use rlua::Table;
 
 use crate::Context;
 
+use self::template::ArgIndex;
+
 mod serde;
+pub mod template;
 
 pub struct Script<'a> {
     pub lua_module_name: &'a str,
@@ -113,6 +116,47 @@ impl<'a> Script<'a> {
     }
 }
 
+fn module_path(module_name: &str) -> String {
+    format!("modules/{}.lua", module_name)
+}
+
+pub struct ModuleFunction<'a> {
+    pub module_name: &'a str,
+    pub function_name: &'a str,
+}
+
+pub fn call(
+    modfn: &ModuleFunction,
+    args: impl Iterator<Item = (ArgIndex, String)>,
+    ctx: &Context,
+) -> Result<String, ScriptError> {
+    let filename = module_path(modfn.module_name);
+
+    let lua_entr = ctx
+        .branch_head()
+        .unwrap()
+        .tree()
+        .and_then(|tree| tree.get_path(Path::new(&filename)))
+        .map_err(|_| ScriptError::ModuleNotFound)?;
+
+    let lua_blob = ctx.repo.find_blob(lua_entr.id()).unwrap();
+    let lua_code = from_utf8(lua_blob.content()).map_err(|_| ScriptError::ModuleNotUtf8)?;
+
+    lua_context(|ctx| {
+        let module: Table = ctx.load(lua_code).eval()?;
+        let view: Function = module.get(modfn.function_name)?;
+        let lua_args = ctx.create_table()?;
+        for (idx, val) in args {
+            match idx {
+                ArgIndex::Str(s) => lua_args.set(s, val)?,
+                ArgIndex::Num(n) => lua_args.set(n, val)?,
+            }
+        }
+        view.call(lua_args)
+    })
+    .map_err(ScriptError::LuaError)
+}
+
 impl Display for ScriptError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
diff --git a/src/lua/template.rs b/src/lua/template.rs
new file mode 100644
index 0000000..de13481
--- /dev/null
+++ b/src/lua/template.rs
@@ -0,0 +1,333 @@
+use logos::Logos;
+
+use super::ModuleFunction;
+
+#[derive(Logos, Debug, PartialEq)]
+pub(crate) enum Token {
+    #[token("{{")]
+    OpenCall,
+
+    #[token("\\}}")]
+    EscapedCloseCall,
+
+    #[token("}}")]
+    CloseCall,
+
+    #[regex("<pre( +[^>]*)?>")]
+    OpenPre,
+
+    #[regex("</pre>")]
+    ClosePre,
+
+    #[regex("<raw>")]
+    OpenRaw,
+
+    #[regex("</raw>")]
+    CloseRaw,
+
+    #[token("<!--")]
+    OpenComment,
+
+    #[token("-->")]
+    CloseComment,
+
+    #[error]
+    Other,
+}
+
+fn strip_marker(num: usize) -> String {
+    format!("\x7fUNIQ-{:05X}-QINU\x7f", num)
+}
+
+#[derive(Logos, Debug, PartialEq)]
+pub(crate) enum MarkerToken {
+    #[regex("\x7fUNIQ-[A-F0-9][A-F0-9][A-F0-9][A-F0-9][A-F0-9]-QINU\x7f")]
+    StripMarker,
+
+    #[error]
+    Other,
+}
+
+#[derive(Debug)]
+pub enum TokenType {
+    OpenPre,
+    ClosePre,
+    OpenRaw,
+    CloseRaw,
+    OpenCall,
+    CloseCall,
+}
+
+#[derive(Debug)]
+pub(crate) enum Sub<'a> {
+    Call(&'a str),
+    Raw(&'a str),
+    Pre { attrs: &'a str, content: &'a str },
+    StrayToken(TokenType, &'a str),
+}
+
+pub(crate) struct ParserState<'a> {
+    substitutions: Vec<Sub<'a>>,
+}
+
+impl<'a> ParserState<'a> {
+    fn new() -> Self {
+        Self {
+            substitutions: Vec::new(),
+        }
+    }
+
+    fn add(&mut self, sub: Sub<'a>) -> usize {
+        let idx = self.substitutions.len();
+        self.substitutions.push(sub);
+        idx
+    }
+
+    pub(crate) fn postprocess(
+        &self,
+        text: &str,
+        render: impl Fn(&Sub<'a>, &mut String) -> std::fmt::Result,
+    ) -> String {
+        let mut out = String::new();
+        let mut lex = MarkerToken::lexer(text);
+        while let Some(tok) = lex.next() {
+            match tok {
+                MarkerToken::StripMarker => match usize::from_str_radix(&lex.slice()[6..11], 16) {
+                    Ok(idx) => {
+                        render(&self.substitutions[idx], &mut out);
+                    }
+                    Err(e) => out.push_str(lex.slice()),
+                },
+                MarkerToken::Other => out.push_str(lex.slice()),
+            }
+        }
+        out.push_str(lex.remainder());
+        out
+    }
+}
+
+pub(crate) fn preprocess(text: &str) -> (String, ParserState) {
+    let mut stripped = String::new();
+    let mut lex = Token::lexer(text);
+    let mut state = ParserState::new();
+
+    while let Some(tok) = lex.next() {
+        match tok {
+            Token::OpenComment => {
+                let span = lex.span();
+
+                stripped.push_str(if lex.any(|t| matches!(&t, Token::CloseComment)) {
+                    &lex.source()[span.start..lex.span().end]
+                } else {
+                    &lex.source()[span.start..]
+                });
+            }
+            Token::OpenRaw => {
+                let span = lex.span();
+                let slice = lex.slice();
+
+                stripped.push_str(&strip_marker(state.add(
+                    if lex.any(|t| matches!(&t, Token::CloseRaw)) {
+                        Sub::Raw(&lex.source()[span.end..lex.span().start])
+                    } else {
+                        lex = Token::lexer(&lex.source()[span.end..]);
+                        Sub::StrayToken(TokenType::OpenRaw, slice)
+                    },
+                )));
+            }
+            Token::OpenCall => {
+                let span = lex.span();
+                let slice = lex.slice();
+
+                stripped.push_str(&strip_marker(state.add(
+                    if lex.any(|t| matches!(&t, Token::CloseCall)) {
+                        Sub::Call(&lex.source()[span.end..lex.span().start])
+                    } else {
+                        lex = Token::lexer(&lex.source()[span.end..]);
+                        Sub::StrayToken(TokenType::OpenCall, slice)
+                    },
+                )));
+            }
+            Token::OpenPre => {
+                let span = lex.span();
+                let slice = lex.slice();
+
+                stripped.push_str(&strip_marker(state.add(
+                    if lex.any(|t| matches!(&t, Token::ClosePre)) {
+                        Sub::Pre {
+                            attrs: &lex.source()[span.start + 4..span.end - 1],
+                            content: &lex.source()[span.end..lex.span().start],
+                        }
+                    } else {
+                        lex = Token::lexer(&lex.source()[span.end..]);
+                        Sub::StrayToken(TokenType::OpenPre, slice)
+                    },
+                )));
+            }
+            Token::CloseCall => {
+                stripped.push_str(&strip_marker(
+                    state.add(Sub::StrayToken(TokenType::CloseCall, lex.slice())),
+                ));
+            }
+            Token::ClosePre => {
+                stripped.push_str(&strip_marker(
+                    state.add(Sub::StrayToken(TokenType::ClosePre, lex.slice())),
+                ));
+            }
+            Token::CloseRaw => {
+                stripped.push_str(&strip_marker(
+                    state.add(Sub::StrayToken(TokenType::CloseRaw, lex.slice())),
+                ));
+            }
+            _ => {
+                stripped.push_str(lex.slice());
+            }
+        }
+    }
+    stripped.push_str(lex.remainder());
+
+    (stripped, state)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fmt::Write;
+
+    use crate::lua::template::preprocess;
+
+    use super::Sub;
+
+    fn debug_format(sub: &Sub, out: &mut String) -> std::fmt::Result {
+        write!(out, "{:?}", sub)
+    }
+
+    #[test]
+    fn test_parse() {
+        let (text, state) = preprocess("{{foobar}} <pre lang=js>foo</pre> <raw>{{test</raw>}}");
+        let out = state.postprocess(&text, debug_format);
+        assert_eq!(
+            out,
+            r#"Call("foobar") Pre { attrs: " lang=js", content: "foo" } Raw("{{test")StrayToken(CloseCall, "}}")"#
+        );
+    }
+
+    #[test]
+    fn test_stray_open_tokens() {
+        let (text, state) = preprocess("{{ <pre> <raw>");
+        let out = state.postprocess(&text, debug_format);
+        assert_eq!(
+            out,
+            r#"StrayToken(OpenCall, "{{") StrayToken(OpenPre, "<pre>") StrayToken(OpenRaw, "<raw>")"#
+        );
+    }
+
+    #[test]
+    fn test_stray_close_tokens() {
+        let (text, state) = preprocess("}} </pre> </raw>");
+        let out = state.postprocess(&text, debug_format);
+        assert_eq!(
+            out,
+            r#"StrayToken(CloseCall, "}}") StrayToken(ClosePre, "</pre>") StrayToken(CloseRaw, "</raw>")"#
+        );
+    }
+
+    #[test]
+    fn test_comment() {
+        let (text, _state) =
+            preprocess("<!-- {{foobar}} <pre lang=js>foo</pre> <raw>{{test</raw>}}");
+        assert_eq!(
+            text,
+            r#"<!-- {{foobar}} <pre lang=js>foo</pre> <raw>{{test</raw>}}"#
+        );
+    }
+
+    #[test]
+    fn test_call_after_stray_open() {
+        let (text, state) = preprocess("<pre> {{foo}}");
+        let out = state.postprocess(&text, debug_format);
+        assert_eq!(out, r#"StrayToken(OpenPre, "<pre>") Call("foo")"#);
+    }
+}
+
+#[derive(Logos, Debug, PartialEq)]
+enum CallInnerToken {
+    #[token("\\|")]
+    EscapedPipe,
+
+    #[regex("\\}}")]
+    EscapedClose,
+
+    #[token("|")]
+    Pipe,
+
+    #[error]
+    Other,
+}
+
+#[derive(Debug)]
+pub enum ArgIndex {
+    Str(String),
+    Num(usize),
+}
+
+fn parse_arg(text: &str, cur_idx: &mut usize) -> (ArgIndex, String) {
+    if let Some((key, val)) = text.split_once('=') {
+        let key = key.trim();
+        let idx = match key.parse::<usize>() {
+            Ok(n) => ArgIndex::Num(n),
+            Err(_) => ArgIndex::Str(key.to_string()),
+        };
+        (idx, val.trim().to_string())
+    } else {
+        let ret = (ArgIndex::Num(*cur_idx), text.trim().to_string());
+        *cur_idx += 1;
+        ret
+    }
+}
+
+pub fn parse_call_args(text: &str) -> (ModuleFunction, std::vec::IntoIter<(ArgIndex, String)>) {
+    let mut args = Vec::new();
+    let mut arg = String::new();
+
+    let mut iter = text.splitn(2, '|');
+    let arg0 = iter.next().unwrap();
+
+    let mut idx = 1;
+
+    if let Some(rest) = iter.next() {
+        let mut lexer = CallInnerToken::lexer(rest);
+        while let Some(tok) = lexer.next() {
+            match tok {
+                CallInnerToken::EscapedClose => arg.push_str("}}"),
+                CallInnerToken::EscapedPipe => arg.push('|'),
+                CallInnerToken::Other => arg.push_str(lexer.slice()),
+                CallInnerToken::Pipe => {
+                    args.push(parse_arg(&arg, &mut idx));
+                    arg = String::new();
+                    idx += 1;
+                }
+            }
+        }
+        arg.push_str(lexer.remainder());
+        args.push(parse_arg(&arg, &mut idx));
+    }
+
+    let mut parts = arg0.splitn(2, '.');
+
+    let module_name;
+    let mut function_name = parts.next().unwrap();
+    if let Some(next) = parts.next() {
+        module_name = function_name;
+        function_name = next;
+    } else {
+        module_name = "default";
+    }
+
+    (
+        ModuleFunction {
+            module_name,
+            function_name,
+        },
+        args.into_iter(),
+    )
+}
diff --git a/src/main.rs b/src/main.rs
index 2cd3a5d..e2fea85 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -18,7 +18,6 @@ use hyper::StatusCode;
 use hyper::{Body, Server};
 use origins::HttpOrigin;
 use percent_encoding::percent_decode_str;
-use pulldown_cmark::html;
 use pulldown_cmark::Options;
 use pulldown_cmark::Parser;
 use serde::Deserialize;
@@ -28,6 +27,7 @@ use sputnik::request::SputnikParts;
 use sputnik::response::SputnikBuilder;
 use std::convert::Infallible;
 use std::env;
+use std::fmt::Write;
 use std::path::Path;
 use std::sync::Arc;
 
@@ -516,14 +516,53 @@ enum RenderMode {
     Preview,
 }
 
-fn render_markdown(input: &str, page: &mut Page, _mode: RenderMode) {
-    let parser = Parser::new_ext(input, Options::all());
+fn render_markdown(text: &str, page: &mut Page, _mode: RenderMode, ctx: &Context) {
+    let md_options = Options::all();
+
     page.body.push_str("<div class=markdown-output>");
-    html::push_html(&mut page.body, parser);
+    #[cfg(feature = "lua")]
+    {
+        use lua::template::{parse_call_args, Sub};
+
+        let (text, state) = lua::template::preprocess(text);
+        let parser = Parser::new_ext(&text, md_options);
+        let mut out = String::new();
+        pulldown_cmark::html::push_html(&mut out, parser);
+
+        let out = state.postprocess(&out, |sub, out| match *sub {
+            Sub::Raw(raw) => out.write_str(&html_escape(raw)),
+            Sub::Pre { attrs, content } => {
+                write!(out, "<pre {attrs}>{}</pre>", html_escape(content))
+            }
+            Sub::StrayToken(_, text) => out.write_str(&html_escape(text)),
+            Sub::Call(text) => {
+                let (modfn, args) = parse_call_args(text);
+
+                // TODO: cache module source code to avoid unnecessary git reads for repeated templates
+                match lua::call(&modfn, args, ctx) {
+                    Ok(output) => out.write_str(&output),
+                    Err(err) => write!(
+                        out,
+                        "<span class=error>{}: {}</span>",
+                        html_escape(modfn.module_name), // TODO: link module
+                        &html_escape(err.to_string()),
+                    ),
+                }
+            }
+        });
+        page.body.push_str(&out);
+    }
+    #[cfg(not(feature = "lua"))]
+    {
+        let parser = Parser::new_ext(text, md_options);
+        let mut out = String::new();
+        pulldown_cmark::html::push_html(&mut out, parser);
+        page.body.push_str(&out);
+    }
     page.body.push_str("</div>");
 }
 
-fn get_renderer(path: &Utf8Path) -> Option<fn(&str, &mut Page, RenderMode)> {
+fn get_renderer(path: &Utf8Path) -> Option<fn(&str, &mut Page, RenderMode, ctx: &Context)> {
     match path.extension() {
         Some("md") => Some(render_markdown),
         _ => None,
diff --git a/src/post_routes.rs b/src/post_routes.rs
index 51b35ec..fda1619 100644
--- a/src/post_routes.rs
+++ b/src/post_routes.rs
@@ -361,6 +361,6 @@ async fn preview_edit<C: Controller>(
     let form: EditForm = body.into_form().await?;
     let new_text = form.text.replace("\r\n", "\n");
     let mut page = edit_text_form(&form, None, controller, &ctx, parts);
-    get_renderer(&ctx.path).unwrap()(&new_text, &mut page, RenderMode::Preview);
+    get_renderer(&ctx.path).unwrap()(&new_text, &mut page, RenderMode::Preview, &ctx);
     Ok(page.into())
 }