Pass through Lua code in script

core/inputs-texlike.lua

@@ -1,49 +1,90 @@
 SILE.inputs.TeXlike = {}
-local epnf = require( "epnf" )
+local epnf = require("epnf")
 
 local ID = lpeg.C( SILE.parserBits.letter * (SILE.parserBits.letter+SILE.parserBits.digit)^0 )
-SILE.inputs.TeXlike.identifier = (ID + lpeg.P("-") + lpeg.P(":"))^1
+SILE.inputs.TeXlike.identifier = (ID + lpeg.P"-" + lpeg.P":")^1
 
 SILE.inputs.TeXlike.parser = function (_ENV)
   local _ = WS^0
-  local sep = lpeg.S(",;") * _
-  local quotedString = (P("\"") * C((1-lpeg.S("\""))^1) * P("\""))
-  local value = (quotedString + (1-lpeg.S(",;]"))^1 )
-  local myID = C( SILE.inputs.TeXlike.identifier + lpeg.P(1) ) / function (t) return t end
-  local pair = lpeg.Cg(myID * _ * "=" * _ * C(value)) * sep^-1   / function (...) local t= {...}; return t[1], t[#t] end
-  local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
-  local parameters = (P("[") * list * P("]")) ^-1 / function (a) return type(a)=="table" and a or {} end
-  local anything = C( (1-lpeg.S("\\{}%\r\n")) ^1)
-  local lineEndLineStartSpace = (lpeg.S(" ")^0 * lpeg.S("\r\n")^1 * lpeg.S(" ")^0)^-1
-  local comment = ((P("%") * (1-lpeg.S("\r\n"))^0 * lpeg.S("\r\n")^-1) /function () return "" end)
+  local sep = S",;" * _
+  local eol = S"\r\n"
+  local quote = P'"'
+  local quotedString = ( quote * C((1-quote)^1) * quote )
+  local value = ( quotedString + (1-S",;]")^1 )
+  local myID = C(SILE.inputs.TeXlike.identifier + P(1)) / 1
+  local pair = Cg(myID * _ * "=" * _ * C(value)) * sep^-1 / function (...) local t = {...}; return t[1], t[#t] end
+  local list = Cf(Ct"" * pair^0, rawset)
+  local parameters = (
+      P"[" *
+      list *
+      P"]"
+    )^-1/function (a) return type(a)=="table" and a or {} end
+  local comment = (
+      P"%" *
+      P(1-eol)^0 *
+      eol^-1
+    ) / ""
 
   START "document"
-  document = V("stuff") * (-1 + E("Unexpected character at end of input"))
-  text = C( (1-lpeg.S("\\{}%")) ^1)
-  stuff = Cg(V"environment" +
-    comment
-    + V("text") + V"bracketed_stuff" + V"command")^0
-  bracketed_stuff = P"{" * V"stuff" * (P"}" + E("} expected"))
-  command =((P("\\")-P("\\begin")) * Cg(myID, "tag") * Cg(parameters,"attr") * V"bracketed_stuff"^0)-P("\\end{")
+  document = V"texlike_stuff" * EOF"Unexpected character at end of input"
+  texlike_stuff = Cg(
+      V"environment" +
+      comment +
+      V"texlike_text" +
+      V"texlike_bracketed_stuff" +
+      V"command"
+    )^0
+  passthrough_stuff = C(Cg(
+      V"passthrough_text" +
+      V"passthrough_debracketed_stuff"
+    )^0)
+  passthrough_env_stuff = Cg(
+      V"passthrough_env_text"
+    )^0
+  texlike_text = C((1-S("\\{}%"))^1)
+  passthrough_text = C((1-S("{}"))^1)
+  passthrough_env_text = C((1-(P"\\end{" * (myID * Cb"tag") * P"}"))^1)
+  texlike_bracketed_stuff = P"{" * V"texlike_stuff" * ( P"}" + E("} expected") )
+  passthrough_bracketed_stuff = P"{" * V"passthrough_stuff" * ( P"}" + E("} expected") )
+  passthrough_debracketed_stuff = C(V"passthrough_bracketed_stuff")
+  command = (
+      ( P"\\"-P"\\begin" ) *
+      Cg(myID, "tag") *
+      Cg(parameters,"attr") *
+      (
+        (Cmt(Cb"tag", function(_, _, tag) return tag == "script" end) * V"passthrough_bracketed_stuff") +
+        (Cmt(Cb"tag", function(_, _, tag) return tag ~= "script" end) * V"texlike_bracketed_stuff")
+      )^0
+    ) - P("\\end{")
   environment =
-    P("\\begin") * Cg(parameters, "attr") * P("{") * Cg(myID, "tag") * P("}")
-      * V("stuff")
-    * (P("\\end{") * (
-      Cmt(myID * Cb("tag"), function(s,i,a,b) return a==b end) +
-      E("Environment mismatch")
-    ) * (P("}") * _) + E("Environment begun but never ended"))
+    P"\\begin" *
+    Cg(parameters, "attr") *
+    P"{" *
+    Cg(myID, "tag") *
+    P"}" *
+    (
+      (Cmt(Cb"tag", function(_, _, tag) return tag == "script" end) * V"passthrough_env_stuff") +
+      (Cmt(Cb"tag", function(_, _, tag) return tag ~= "script" end) * V"texlike_stuff")
+    ) *
+    (
+      P"\\end{" *
+      (
+        Cmt(myID * Cb"tag", function (_,_,thisTag,lastTag) return thisTag == lastTag end) + E"Environment mismatch"
+      ) *
+      ( P"}" * _ ) + E"Environment begun but never ended"
+    )
 end
 
 local linecache = {}
 local lno, col, lastpos
-local function resetCache()
+local function resetCache ()
   lno = 1
   col = 1
   lastpos = 0
   linecache = { { lno = 1, pos = 1} }
 end
 
-local function getline( s, p )
+local function getline (s, p)
   start = 1
   lno = 1
   if p > lastpos then
@@ -71,34 +112,41 @@ local function getline( s, p )
   return lno, col
 end
 
-local function massage_ast(t,doc)
+local function massage_ast (tree, doc)
   -- Sort out pos
-  if type(t) == "string" then return t end
-  if t.pos then
-    t.line, t.col = getline(doc, t.pos)
+  if type(tree) == "string" then return tree end
+  if tree.pos then
+    tree.line, tree.col = getline(doc, tree.pos)
   end
-  if t.id == "document" then return massage_ast(t[1],doc) end
-  if t.id == "text" then return t[1] end
-  if t.id == "bracketed_stuff" then return massage_ast(t[1],doc) end
-  for k,v in ipairs(t) do
-    if v.id == "stuff" then
-      local val = massage_ast(v,doc)
-      SU.splice(t, k,k, val)
+  if tree.id == "document"
+      or tree.id == "texlike_bracketed_stuff"
+      or tree.id == "passthrough_bracketed_stuff"
+    then return massage_ast(tree[1], doc) end
+  if tree.id == "texlike_text"
+      or tree.id == "passthrough_text"
+      or tree.id == "passthrough_env_text"
+    then return tree[1] end
+  for key, val in ipairs(tree) do
+    if val.id == "texlike_stuff"
+      or val.id == "passthrough_stuff"
+      or val.id == "passthrough_env_stuff"
+      then
+      SU.splice(tree, key, key, massage_ast(val, doc))
     else
-      t[k] = massage_ast(v,doc)
+      tree[key] = massage_ast(val, doc)
     end
   end
-  return t
+  return tree
 end
 
-function SILE.inputs.TeXlike.process(doc)
+function SILE.inputs.TeXlike.process (doc)
   local tree = SILE.inputs.TeXlike.docToTree(doc)
   local root = SILE.documentState.documentClass == nil
   if root then
     if tree.tag == "document" then
       SILE.inputs.common.init(doc, tree)
       SILE.process(tree)
-    elseif pcall(function() assert(loadstring(doc))() end) then
+    elseif pcall(function () assert(loadstring(doc))() end) then
     else
       SU.error("Input not recognized as Lua or SILE content")
     end
@@ -110,20 +158,20 @@ end
 
 local _parser
 
-function SILE.inputs.TeXlike.rebuildParser()
+function SILE.inputs.TeXlike.rebuildParser ()
   _parser = epnf.define(SILE.inputs.TeXlike.parser)
 end
 
 SILE.inputs.TeXlike.rebuildParser()
 
-function SILE.inputs.TeXlike.docToTree(doc)
+function SILE.inputs.TeXlike.docToTree (doc)
   local tree = epnf.parsestring(_parser, doc)
-  -- a document always consists of one stuff
+  -- a document always consists of one texlike_stuff
   tree = tree[1][1]
-  if tree.id == "text" then tree = {tree} end
+  if tree.id == "texlike_text" then tree = {tree} end
   if not tree then return end
   resetCache()
-  tree = massage_ast(tree,doc)
+  tree = massage_ast(tree, doc)
   return tree
 end
 

documentation/sile.sil

@@ -966,6 +966,22 @@ produces the following output:
 \end{script}
 \line
 \end{examplefont}
+
+There is one notable caveat when embedding Lua code documents written with the
+TeX-flavor markup. Since SILE has to first parse the TeX markup to find the
+start and end of such script commands \em{without} understanding what's in
+between, it is strictly necessary that no end tags appear inside the Lua code.
+This means that for the environment block version (\code{\\begin\{script\}})
+there must be no instances of \code{\\end\{script\}} inside the Lua code block,
+even inside a string that would otherwise be valid Lua code, (e.g. if it was
+inside a quoted string or Lua comment). The first instance of such and end tag
+terminates the block and there is currently no way to escape it. For the inline
+comand form (\code{\\script}) an exact matching number of open and closing
+braces may appear in the Lua code — the next closing brace at the same level as
+the opening brace will close the tag, even if it happens to be inside some
+quoted string in the Lua code. Including any nuber of nested opening and closing
+braces is possible as long as they are balanced.
+
 \chapter{SILE Packages}
 
 SILE comes with a number of packages which provide additional functionality.

tests/inline-lua.expected

@@ -0,0 +1,154 @@
+Set paper size 	419.5275636	595.275597
+Begin page
+Mx 	54.8208
+My 	76.6203
+Set font 	Gentium Plus;10;400;;normal;;LTR
+T	49 82 85 80 68 79	(Normal)
+Mx 	88.2817
+T	69 85 68 70 78 72 87 72 71	(bracketed)
+Mx 	131.6888
+T	8	(%)
+Mx 	141.2386
+T	80 68 85 78	(mark)
+Mx 	165.3488
+T	68 81 71	(and)
+Mx 	183.3068
+T	83 88 85 72	(pure)
+Mx 	205.1319
+T	70 82 80 80 68 81 71	(command)
+Mx 	248.5537
+Set font 	Gentium Plus;10;400;Italic;normal;;LTR
+T	20 19 19	(100)
+Mx 	260.6094
+T	8	(%)
+Mx 	269.4397
+T	76 87 68 79 76 70	(italic)
+Mx 	288.1652
+Set font 	Gentium Plus;10;400;;normal;;LTR
+T	17	(.)
+Mx 	54.8208
+My 	88.6203
+T	48 82 71 88 79 88 86	(Modulus)
+Mx 	92.9373
+T	73 85 82 80	(from)
+Mx 	115.7267
+T	72 81 89 76 85 82 81 80 72 81 87	(environment)
+Mx 	169.6231
+T	29	(:)
+Mx 	174.5707
+T	22	(3)
+Mx 	181.9206
+T	13	(*)
+Mx 	189.0459
+T	22	(3)
+Mx 	196.3958
+T	8	(%)
+Mx 	205.9528
+T	24	(5)
+Mx 	213.3027
+T	32	(=)
+Mx 	220.3498
+T	23	(4)
+Mx 	54.8208
+My 	100.6203
+T	48 82 71 88 79 88 86	(Modulus)
+Mx 	92.9371
+T	76 81	(in)
+Mx 	103.8221
+T	76 81 79 76 81 72	(inline)
+Mx 	130.2636
+T	70 82 80 80 68 81 71	(command)
+Mx 	171.0351
+T	29	(:)
+Mx 	175.9825
+T	22	(3)
+Mx 	183.3322
+T	13	(*)
+Mx 	190.4574
+T	22	(3)
+Mx 	197.8071
+T	8	(%)
+Mx 	207.3639
+T	24	(5)
+Mx 	214.7137
+T	32	(=)
+Mx 	221.7607
+T	23	(4)
+Mx 	54.8208
+My 	112.6203
+T	37 85 68 70 72 86	(Braces)
+Mx 	84.3676
+T	76 81	(in)
+Mx 	95.2425
+T	76 81 79 76 81 72	(inline)
+Mx 	121.6740
+T	70 82 80 80 68 81 71	(command)
+Mx 	162.4455
+T	29	(:)
+Mx 	167.3829
+T	48 68 87 70 75 76 81 74	(Matching)
+Mx 	209.1172
+T	69 85 68 70 72	(brace)
+Mx 	234.3818
+T	94	({)
+Mx 	237.7411
+T	96	(})
+Mx 	243.7479
+T	76 81 70 72 83 87 76 82 81	(inception)
+Mx 	285.6091
+T	76 86	(is)
+Mx 	294.8288
+T	89 68 79 76 71	(valid)
+Mx 	314.9704
+T	17	(.)
+Mx 	54.8208
+My 	124.8508
+T	47 88 68	(Lua)
+Mx 	72.1179
+T	70 75 68 85 68 70 87 72 85	(character)
+Mx 	114.1855
+T	72 86 70 68 83 72 86	(escapes)
+Mx 	148.0548
+T	73 85 82 80	(from)
+Mx 	170.8353
+T	72 81 89 76 85 82 81 80 72 81 87	(environment)
+Mx 	224.7318
+T	29	(:)
+Mx 	229.6704
+T	63	(\)
+Mx 	234.3628
+T	69 68 70 78 86 79 68 86 75	(backslash)
+Mx 	276.4695
+T	9	(&)
+Mx 	285.7880
+T	0	(	)
+Mx 	292.6240
+T	87 68 69	(tab)
+Mx 	54.8208
+My 	136.9836
+T	47 88 68	(Lua)
+Mx 	72.1193
+T	70 75 68 85 68 70 87 72 85	(character)
+Mx 	114.1883
+T	72 86 70 68 83 72 86	(escapes)
+Mx 	148.0591
+T	73 85 82 80	(from)
+Mx 	170.8410
+T	70 82 80 80 68 81 71	(command)
+Mx 	211.6125
+T	29	(:)
+Mx 	216.5527
+T	63	(\)
+Mx 	221.2450
+T	69 68 70 78 86 79 68 86 75	(backslash)
+Mx 	263.3531
+T	9	(&)
+Mx 	272.6731
+T	0	(	)
+Mx 	279.5091
+T	87 68 69	(tab)
+Mx 	195.4611
+My 	519.8019
+T	20	(1)
+End page
+Finish

tests/inline-lua.sil

@@ -0,0 +1,18 @@
+\begin[papersize=a5,class=book]{document}
+% A real comment
+\begin{script}
+answer = { output = tostring(3 * 3 % 5) }
+escapes = "\\backslash & \ttab"
+\end{script}
+Normal {bracketed \% mark} and pure command \em{100\% italic}.
+
+Modulus from environment: 3 * 3 \% 5 = \script{SILE.typesetter:typeset(answer.output)}
+
+Modulus in inline command: 3 * 3 \% 5 = \script{SILE.typesetter:typeset(tostring(3 * 3 % 5))}
+
+Braces in inline command: \script{val = { str = "Matching brace {} inception is" }; SILE.typesetter:typeset(val.str)} valid.
+
+Lua character escapes from environment: \script{SILE.typesetter:typeset(escapes)}
+
+Lua character escapes from command: \script{SILE.typesetter:typeset("\\backslash & \ttab")}
+\end{document}