Implement a basic Roc test runner

Dockerfile

@@ -1,8 +1,23 @@
-FROM alpine:3.18
+FROM debian:latest
+
+RUN apt-get update --fix-missing
+RUN apt-get upgrade --yes
 
 # install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apt-get install --yes wget jq coreutils libc-dev binutils \
+    && apt clean \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt/test-runner
+COPY bin/download-basic-cli.roc bin/download-basic-cli.roc
+
+# download & install roc and the basic-cli platform
+RUN wget -q -O roc.tar.gz https://github.com/roc-lang/roc/releases/download/nightly/roc_nightly-linux_x86_64-latest.tar.gz \
+    && mkdir /usr/lib/roc \
+    && tar -xvz -f roc.tar.gz --directory /usr/lib/roc --strip-components=1 \
+    && rm roc.tar.gz \
+    && /usr/lib/roc/roc test bin/download-basic-cli.roc
+ENV PATH="$PATH:/usr/lib/roc"
+
 COPY . .
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

bin/download-basic-cli.roc

@@ -0,0 +1,8 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect Bool.true
+
+main =
+    Task.ok {}

bin/run.sh

@@ -33,9 +33,7 @@ echo "${slug}: testing..."
 
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
+test_output=$(roc test "${solution_dir%/}/${slug}-test.roc" 2>&1)
 
 # Write the results.json file based on the exit code of the command that was 
 # just executed that tested the implementation file
@@ -45,16 +43,22 @@ else
     # OPTIONAL: Sanitize the output
     # In some cases, the test output might be overly verbose, in which case stripping
     # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
+    sanitized_test_output=$(printf "${test_output}\n" | sed 's/ passed in [0-9]\+ ms./ passed./g')
 
     # OPTIONAL: Manually add colors to the output to help scanning the output for errors
     # If the test output does not contain colors to help identify failing (or passing)
     # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
+    # colorized_test_output=$(echo "${sanitized_test_output}" \
     #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
     #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
 
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+    printf "${sanitized_test_output}" | grep -q -E "── EXPECT (FAILED|PANICKED) in "
+    if [ $? -eq 0 ]; then
+        roc_status="fail"
+    else
+        roc_status="error"
+    fi
+    jq -n --arg output "${sanitized_test_output}" "{version: 1, status: \"$roc_status\", message: \$output}" > ${results_file}
 fi
 
 echo "${slug}: done"

tests/all-fail/all-fail-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect 2 + 2 == 5
+expect "four" == "five"
+
+main =
+    Task.ok {}

tests/all-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "\u001b[1;36m── EXPECT FAILED in tests/all-fail/all-fail-test.roc ───────────────────────────\u001b[0m\n\nThis expectation failed:\n\n\u001b[1;36m5\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mexpect 2 + 2 == 5\u001b[0m\n    \u001b[1;31m^^^^^^^^^^^^^^^^^\u001b[0m\n\n\n\u001b[1;36m── EXPECT FAILED in tests/all-fail/all-fail-test.roc ───────────────────────────\u001b[0m\n\nThis expectation failed:\n\n\u001b[1;36m6\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mexpect \"four\" == \"five\"\u001b[0m\n    \u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\n\n\u001b[1;31m2\u001b[0m failed and \u001b[1;32m0\u001b[0m passed."
 }

tests/empty-file/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
-  "status": "fail",
-  "message": "TODO: replace with correct output"
+  "status": "error",
+  "message": "\u001b[1;36m── MISSING HEADER in tests/empty-file/empty-file-test.roc ──────────────────────\u001b[0m\n\nI am expecting a header, but got stuck here:\n\n\u001b[1;36m1\u001b[0m\u001b[1;36m│\u001b[0m\n    \u001b[1;31m^\u001b[0m\n\nI am expecting a module keyword next, one of \u001b[1;32minterface\u001b[0m, \u001b[1;32mapp\u001b[0m, \u001b[1;32mpackage\u001b[0m\nor \u001b[1;32mplatform\u001b[0m."
 }

tests/partial-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "\u001b[1;36m── EXPECT FAILED in tests/partial-fail/partial-fail-test.roc ───────────────────\u001b[0m\n\nThis expectation failed:\n\n\u001b[1;36m6\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mexpect \"hello\" == \"good bye\"\u001b[0m\n    \u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\n\n\u001b[1;31m1\u001b[0m failed and \u001b[1;32m1\u001b[0m passed."
 }

tests/partial-fail/partial-fail-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect 2 + 2 == 4
+expect "hello" == "good bye"
+
+main =
+    Task.ok {}

tests/success/success-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect 2 + 2 == 4
+expect "four" == "four"
+
+main =
+    Task.ok {}

tests/syntax-error/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "error",
-  "message": "TODO: replace with correct output"
+  "message": "\u001b[1;36m── NOT END OF FILE in tests/syntax-error/syntax-error-test.roc ─────────────────\u001b[0m\n\nI expected to reach the end of the file, but got stuck here:\n\n\u001b[1;36m3\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37miMpOrT pf.Task exposing [Task]\u001b[0m\n    \u001b[1;31m^\u001b[0m"
 }

tests/syntax-error/syntax-error-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+iMpOrT pf.Task exposing [Task]
+
+expect 2 + 2 == 5
+expect "four" == "five"
+
+main =
+    Task.ok {}