Add Lua and Databroker support

.gitattributes

@@ -0,0 +1 @@
+* text=auto

.luarc.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://raw.githubusercontent.com/sumneko/vscode-lua/master/setting/schema.json",
+  "workspace.library": ["engine/lua"],
+  "runtime.version": "Lua 5.4",
+  "completion.displayContext": 1,
+  "diagnostics.globals": ["cloe"],
+  "hint.enable": true
+}

.vscode/c_cpp_properties.json

@@ -0,0 +1,18 @@
+{
+    "configurations": [
+        {
+            "name": "Linux",
+            "includePath": [
+                "${workspaceFolder}/**"
+            ],
+            "defines": [],
+            "compilerPath": "/usr/bin/g++",
+            "cStandard": "c17",
+            "cppStandard": "c++17",
+            "intelliSenseMode": "linux-gcc-x64",
+            //run `make configure` to create the file
+            "compileCommands": "${workspaceFolder}/compile_commands.json"
+        }
+    ],
+    "version": 4
+}

.vscode/launch.json

@@ -0,0 +1,76 @@
+{
+    "configurations": [
+        {
+            "name": "Debug cloe",
+            "type": "cppdbg",
+            "request": "launch",
+            "program": "${input:get_executable_path}",
+            "args": [
+                "run",
+                "--write-output",
+                "${workspaceFolder}/${input:get_test_case}",
+                //Example to run a specific test-case
+                //"${workspaceFolder}/tests/test_lua04_schedule_test.lua",
+                //Example to run a specific smoketest
+                //"-u",
+                //"smoketest",
+                //"${workspaceFolder}/tests/test_engine_replica_smoketest_temp.json",
+            ],
+            "stopAtEntry": false,
+            "cwd": "${fileDirname}",
+            "environment": [],
+            "externalConsole": false,
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                },
+                {
+                    "description": "Set Disassembly Flavor to Intel",
+                    "text": "-gdb-set disassembly-flavor intel",
+                    "ignoreFailures": true
+                }
+            ],
+            "preLaunchTask": "Build Cloe",
+            "miDebuggerPath": "/usr/bin/gdb",
+            "envFile": "${input:get_shell_path}"
+        }
+    ],
+    "inputs": [
+        {
+            "type": "command",
+            "id": "get_test_case",
+            "command": "shellCommand.execute",
+            "args":{
+                "command":"ls tests/*.lua",
+                "cwd": "${workspaceFolder}"
+            }
+        },
+        {
+            "type": "command",
+            "id": "get_shell_path",
+            "command": "shellCommand.execute",
+            //bash magic to set CLOE_SHELL variable and get the value of it
+            "args": {
+                "command": "bash -c \"source <(PYTHONPATH=\"${workspaceFolder}/cli\" python3 -m cloe_launch activate tests/conanfile_all.py) && printenv CLOE_SHELL\"",
+                "cwd": "${workspaceFolder}",
+                "useSingleResult": "true"
+            }
+        },
+        {
+            "type": "command",
+            "id": "get_executable_path",
+            "command": "shellCommand.execute",
+            //bash magic to set CLOE_ENGINE variable and get the value of it
+            "args": {
+                //"command": "bash -c \"source <(cloe-launch activate tests/conanfile_all.py) && printenv CLOE_ENGINE\"",
+                "command": "bash -c \"source <(PYTHONPATH=\"${workspaceFolder}/cli\" python3 -m cloe_launch activate tests/conanfile_all.py) && printenv CLOE_ENGINE\"",
+                "cwd": "${workspaceFolder}",
+                "useSingleResult": "true"
+            }
+        }
+    ],
+    "version": "2.0.0"
+}

.vscode/tasks.json

@@ -0,0 +1,41 @@
+{
+    "tasks": [
+        {
+            "type": "shell",
+            "label": "Build Cloe",
+            "command": "make",
+            "args": [
+                "all",
+                "BUILD_TYPE=Debug"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}"
+            },
+            "problemMatcher": [
+                "$gcc"
+            ],
+            "group": "build",
+            "detail": "Task generated by Debugger."
+        },
+        {
+            "type": "shell",
+            "label": "Build Cloe (without tests)",
+            "command": "make",
+            "args": [
+                "all",
+                "CONAN_OPTIONS=\"-c tools.build:skip_test=1\"",
+                "CONAN_RUN_TESTS=0",
+                "BUILD_TYPE=Debug"
+            ],
+            "options": {
+                "cwd": "${workspaceFolder}"
+            },
+            "problemMatcher": [
+                "$gcc"
+            ],
+            "group": "build",
+            "detail": "Task generated by Debugger."
+        },
+    ],
+    "version": "2.0.0"
+}

Makefile

@@ -153,8 +153,13 @@ todos:
 
 .PHONY: grep-uuids
 grep-uuids:
-	$(AG) "\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
+	 @rg -H --no-heading '^.*([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}).*$$' -r '$$1' \
+		 | sed -nr 's/^(.+):([a-f0-9-]+)$$/\1\t\2/p' \
+		 | sort -k2 \
+		 | uniq --all-repeated=separate -f1 \
+		 | sed -r 's/^(.*)  (.*)$$/\2\t\1/'
 
+.PHONY: grep-conan-requires
 grep-conan-requires:
 	@rg -t py '^.*requires\(f?["](.+/[0-9]+\.[^)]+)["].*\).*$$' -r '$$1' -I --no-heading --no-line-number | sort | uniq
 

NOTICE.md

@@ -46,6 +46,42 @@ The following third-party libraries are included in the Cloe repository:
   - Website: https://jothepro.github.io/doxygen-awesome-css
   - Source: docs/_vendor/doxygen-awesome
 
+- Typecheck
+  - License: MIT
+  - License-Source: https://github.com/gvvaughan/typecheck/raw/master/LICENSE.md
+  - Website: https://github.com/gvvaughan/typecheck
+  - Source: engine/lua/typecheck.lua
+
+- Tableshape
+  - License: MIT
+  - License-Source: https://github.com/leafo/tableshape/blob/v2.6.0/README.md
+  - Website: https://github.com/leafo/tableshape
+  - Source: engine/lua/tableshape.lua
+
+- Linenoise
+  - License: BSD2
+  - License-Source: https://raw.githubusercontent.com/antirez/linenoise/master/LICENSE
+  - Website: https://github.com/antirez/linenoise
+  - Source: engine/vendor/linenoise
+
+- Inspect.lua
+  - License: MIT
+  - License-Source: https://raw.githubusercontent.com/kikito/inspect.lua/master/MIT-LICENSE.txt
+  - Website: https://github.com/kikito/inspect.lua
+  - Source: engine/lua/inspect.lua
+
+- LRDB
+  - License: BSL-1.0
+  - License-Source: https://www.boost.org/LICENSE_1_0.txt
+  - Website: https://github.com/satoren/LRDB
+  - Source: engine/vendor/lrdb
+
+- Lust
+  - License: MIT
+  - License-Source: https://raw.githubusercontent.com/bjornbytes/lust/master/LICENSE
+  - Website: https://github.com/bjornbytes/lust
+  - Source: engine/lua/lust.lua
+
 The following third-party libraries are used by this project (these are usually
 installed with the help of Conan):
 
@@ -93,7 +129,7 @@ installed with the help of Conan):
   - Conan-Package: inja
 
 - Boost
-  - License: Boost
+  - License: BSL-1.0
   - License-Source: https://www.boost.org/LICENSE_1_0.txt
   - Website: https://www.boost.org
   - Conan-Package: boost

conanfile.py

@@ -50,6 +50,7 @@ class Cloe(ConanFile):
         "fPIC": [True, False],
         "fable_allow_comments": [True, False],
         "engine_server": [True, False],
+        "engine_lrdb": [True, False],
         "with_esmini": [True, False],
         "with_vtd": [True, False],
     }
@@ -58,6 +59,7 @@ class Cloe(ConanFile):
         "fPIC": True,
         "fable_allow_comments": True,
         "engine_server": True,
+        "engine_lrdb": True,
         "with_esmini": True,
         "with_vtd": False,
     }
@@ -72,7 +74,9 @@ class Cloe(ConanFile):
 
         "fable/examples/*",
 
+        "engine/lua/*",
         "engine/webui/*",
+        "engine/vendor/*",
 
         "CMakelists.txt"
     ]
@@ -93,6 +97,7 @@ def requirements(self):
         self.requires("incbin/cci.20211107"),
         self.requires("inja/3.4.0")
         self.requires("nlohmann_json/3.11.3")
+        self.requires("sol2/3.3.1")
         self.requires("spdlog/1.11.0")
         if self.options.engine_server:
             self.requires("oatpp/1.3.0", private=True)
@@ -105,7 +110,6 @@ def requirements(self):
 
     def build_requirements(self):
         self.test_requires("gtest/1.14.0")
-        self.test_requires("sol2/3.3.0")
 
     def layout(self):
         cmake.cmake_layout(self)
@@ -132,6 +136,7 @@ def generate(self):
         tc.cache_variables["CLOE_VERSION"] = self.version
         tc.cache_variables["CLOE_VERSION_U32"] = version_u32
         tc.cache_variables["CLOE_ENGINE_WITH_SERVER"] = self.options.engine_server
+        tc.cache_variables["CLOE_ENGINE_WITH_LRDB"] = self.options.engine_lrdb
         tc.cache_variables["CLOE_WITH_ESMINI"] = self.options.with_esmini
         tc.cache_variables["CLOE_WITH_VTD"] = self.options.with_vtd
         tc.generate()
@@ -195,14 +200,18 @@ def package_info(self):
             self.cpp_info.builddirs.append(os.path.join(self.source_folder, "cmake"))
             self.cpp_info.includedirs.append(os.path.join(self.build_folder, "include"))
             bindir = os.path.join(self.build_folder, "bin")
+            luadir = os.path.join(self.source_folder, "engine/lua")
             libdir = os.path.join(self.build_folder, "lib");
         else:
             self.cpp_info.builddirs.append(os.path.join("lib", "cmake", "cloe"))
             bindir = os.path.join(self.package_folder, "bin")
+            luadir = os.path.join(self.package_folder, "lib/cloe/lua")
             libdir = None
 
         self.output.info(f"Appending PATH environment variable: {bindir}")
         self.runenv_info.prepend_path("PATH", bindir)
+        self.output.info(f"Appending CLOE_LUA_PATH environment variable: {luadir}")
+        self.runenv_info.prepend_path("CLOE_LUA_PATH", luadir)
         if libdir is not None:
             self.output.info(f"Appending LD_LIBRARY_PATH environment variable: {libdir}")
             self.runenv_info.append_path("LD_LIBRARY_PATH", libdir)

docs/reference.rst

@@ -47,6 +47,9 @@ background knowledge for any of the following topics.
 :doc:`reference/plugins`
    provides an overview of all plugins that are part of the Cloe distribution.
 
+:doc:`reference/lua-initialization`
+   provides an overview of how the engine processes a Lua file.
+
 .. toctree::
    :hidden:
 
@@ -62,3 +65,4 @@ background knowledge for any of the following topics.
    reference/events
    reference/actions
    reference/plugins
+   reference/lua-initialization

docs/reference/lua-initialization.md

@@ -0,0 +1,38 @@
+Lua Initialization
+------------------
+
+When a Lua file or script is loaded, the Cloe engine provides a preloaded
+`cloe` table with a large API. This API is defined in part through a Lua
+runtime, and in part from the C++ engine itself.
+
+The following operations occur when the engine runs a simulation defined
+by a Lua file: `cloe-engine run simulation.lua`
+
+1. Read options from the command line and environment:
+
+   - Lua package path (`--lua-path`, `CLOE_LUA_PATH`)
+   - Disable system packages (`--no-system-lua`)
+   - Enable LRDB Lua debugger (`--debug-lua`)
+   - Cloe plugins (`--plugin-path`, `CLOE_PLUGIN_PATH`)
+
+2. Initialize Cloe Stack
+
+   - Load plugins found in plugin path
+
+3. Initialize Lua
+
+   - Set lua package path
+   - Load built-in Lua base libraries (e.g. `os`, `string`)
+   - Expose Cloe API via `cloe` Lua table
+   - Load Cloe Lua runtime (located in the package `lib/cloe/lua` directory)
+
+4. Start LRDB Lua debugger (Optional)
+
+5. Source input files
+
+   - Files ending with `.lua` are merged as Lua
+   - Other files are read as JSON
+
+6. Start simulation
+
+   - Schedule triggers pending from the Lua script

docs/usage.rst

@@ -16,6 +16,11 @@ guides will give you a better feel for how Cloe works and how it is built up.
    usage/viewing-cloe-registry
    usage/running-cloe-webui
    usage/creating-a-stackfile
+
+   usage/lua-introduction
+   usage/lua-cloe-shell
+   usage/lua-editor-integration
+
    usage/configuring-plugins-in-stackfiles
    usage/writing-modular-stackfiles
    usage/user-cloe-configuration