Implement a basic Roc test runner

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   build:
     name: Tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332

.gitignore

@@ -1 +1 @@
-/tests/**/*/results.json
+/tests/**/results.json

Dockerfile

@@ -1,8 +1,28 @@
-FROM alpine:3.18
+FROM ubuntu:24.04
+
+RUN apt-get update --fix-missing
+RUN apt-get upgrade --yes
 
 # install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+RUN apt-get install --yes wget jq coreutils libc-dev binutils \
+    && apt clean \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt/test-runner
+COPY bin/download-basic-cli.roc bin/download-basic-cli.roc
+
+# download & install roc and the basic-cli platform
+RUN wget -q -O roc.tar.gz https://github.com/roc-lang/roc/releases/download/nightly/roc_nightly-linux_x86_64-latest.tar.gz \
+    && mkdir /usr/lib/roc \
+    && tar -xzf roc.tar.gz --directory /usr/lib/roc --strip-components=1 \
+    && rm roc.tar.gz \
+    && wget -q -O roc.tgz https://github.com/ageron/prototype-roc-test-runner/raw/main/bin/roc-release.tgz \
+    && tar -xzf roc.tgz \
+    && rm roc.tgz \
+    && mv -f ./roc /usr/lib/roc/roc \
+    && /usr/lib/roc/roc test bin/download-basic-cli.roc
+
+ENV PATH="$PATH:/usr/lib/roc"
+
 COPY . .
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

bin/download-basic-cli.roc

@@ -0,0 +1,8 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect Bool.true
+
+main =
+    Task.ok {}

bin/run-in-docker.sh

@@ -42,5 +42,5 @@ docker run \
     --read-only \
     --mount type=bind,src="${solution_dir}",dst=/solution \
     --mount type=bind,src="${output_dir}",dst=/output \
-    --mount type=tmpfs,dst=/tmp \
+    --mount type=volume,dst=/tmp \
     exercism/roc-test-runner "${slug}" /solution /output 

bin/run-tests-in-docker.sh

@@ -24,7 +24,7 @@ docker run \
     --network none \
     --read-only \
     --mount type=bind,src="${PWD}/tests",dst=/opt/test-runner/tests \
-    --mount type=tmpfs,dst=/tmp \
+    --mount type=volume,dst=/tmp \
     --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
     --workdir /opt/test-runner \
     --entrypoint /opt/test-runner/bin/run-tests.sh \

bin/run.sh

@@ -33,9 +33,8 @@ echo "${slug}: testing..."
 
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
+
+test_output=$(roc test "${solution_dir%/}/${slug}-test.roc" 2>&1)
 
 # Write the results.json file based on the exit code of the command that was 
 # just executed that tested the implementation file
@@ -45,16 +44,22 @@ else
     # OPTIONAL: Sanitize the output
     # In some cases, the test output might be overly verbose, in which case stripping
     # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
+    sanitized_test_output=$(printf "${test_output}\n" | sed 's/ in [0-9]\+ ms//g')
 
     # OPTIONAL: Manually add colors to the output to help scanning the output for errors
     # If the test output does not contain colors to help identify failing (or passing)
     # tests, it can be helpful to manually add colors to the output
-    # colorized_test_output=$(echo "${test_output}" \
+    # colorized_test_output=$(echo "${sanitized_test_output}" \
     #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
     #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
 
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+    printf "${sanitized_test_output}" | grep -q -E "── EXPECT (FAILED|PANICKED) in "
+    if [ $? -eq 0 ]; then
+        roc_status="fail"
+    else
+        roc_status="error"
+    fi
+    jq -n --arg output "${sanitized_test_output}" "{version: 1, status: \"$roc_status\", message: \$output}" > ${results_file}
 fi
 
 echo "${slug}: done"

tests/all-fail/all-fail-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect 2 + 2 == 5
+expect "four" == "five"
+
+main =
+    Task.ok {}

tests/all-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "\n\u001b[1;36m── BACKPASSING DEPRECATED in ...ejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE/Task.roc ─\u001b[0m\n\nBackpassing (\u001b[1;33m<-\u001b[0m) like this will soon be deprecated:\n\n\u001b[1;36m213\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37m        f <- next |> await\u001b[0m\n              \u001b[1;33m^^^^^^^^^^^^^^^^^^\u001b[0m\n\nYou should use a \u001b[1;33m!\u001b[0m for awaiting tasks or a \u001b[1;33m?\u001b[0m for trying results, and\nfunctions everywhere else.\n\n\n\u001b[1;36m── BACKPASSING DEPRECATED in ...ejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE/Task.roc ─\u001b[0m\n\nBackpassing (\u001b[1;33m<-\u001b[0m) like this will soon be deprecated:\n\n\u001b[1;36m231\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37m        value <- task |> await\u001b[0m\n              \u001b[1;33m^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\nYou should use a \u001b[1;33m!\u001b[0m for awaiting tasks or a \u001b[1;33m?\u001b[0m for trying results, and\nfunctions everywhere else.\n\n\n\u001b[1;36m── BACKPASSING DEPRECATED in ...O2HVVuPJFkwvvsfW6LojkLR20kTVE/InternalHttp.roc ─\u001b[0m\n\nBackpassing (\u001b[1;33m<-\u001b[0m) like this will soon be deprecated:\n\n\u001b[1;36m75\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37m    fmt <- Inspect.custom\u001b[0m\n         \u001b[1;33m^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\nYou should use a \u001b[1;33m!\u001b[0m for awaiting tasks or a \u001b[1;33m?\u001b[0m for trying results, and\nfunctions everywhere else.\n\n\u001b[1;36m────────────────────────────────────────────────────────────────────────────────\u001b[0m\n\n\u001b[1;32m0\u001b[0m errors and \u001b[1;33m3\u001b[0m warnings found\n.\n\nRunning tests…\n\n\u001b[36m────────────────────────────────────────────────────────────────────────────────\u001b[39m\n\u001b[1;36m── EXPECT FAILED in tests/all-fail/all-fail-test.roc ───────────────────────────\u001b[0m\n\nThis expectation failed:\n\n\u001b[1;36m5\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mexpect 2 + 2 == 5\u001b[0m\n    \u001b[1;31m^^^^^^^^^^^^^^^^^\u001b[0m\n\n\n\u001b[1;36m── EXPECT FAILED in tests/all-fail/all-fail-test.roc ───────────────────────────\u001b[0m\n\nThis expectation failed:\n\n\u001b[1;36m6\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mexpect \"four\" == \"five\"\u001b[0m\n    \u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\n\n\u001b[1;31m2\u001b[0m failed and \u001b[1;32m0\u001b[0m passed."
 }

tests/empty-file/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
-  "status": "fail",
-  "message": "TODO: replace with correct output"
+  "status": "error",
+  "message": "\u001b[1;36m── MISSING HEADER in tests/empty-file/empty-file-test.roc ──────────────────────\u001b[0m\n\nI am expecting a header, but got stuck here:\n\n\u001b[1;36m1\u001b[0m\u001b[1;36m│\u001b[0m\n    \u001b[1;31m^\u001b[0m\n\nI am expecting a module keyword next, one of \u001b[1;32minterface\u001b[0m, \u001b[1;32mapp\u001b[0m, \u001b[1;32mpackage\u001b[0m\nor \u001b[1;32mplatform\u001b[0m."
 }

tests/partial-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "\n\u001b[1;36m── BACKPASSING DEPRECATED in ...ejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE/Task.roc ─\u001b[0m\n\nBackpassing (\u001b[1;33m<-\u001b[0m) like this will soon be deprecated:\n\n\u001b[1;36m213\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37m        f <- next |> await\u001b[0m\n              \u001b[1;33m^^^^^^^^^^^^^^^^^^\u001b[0m\n\nYou should use a \u001b[1;33m!\u001b[0m for awaiting tasks or a \u001b[1;33m?\u001b[0m for trying results, and\nfunctions everywhere else.\n\n\n\u001b[1;36m── BACKPASSING DEPRECATED in ...ejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE/Task.roc ─\u001b[0m\n\nBackpassing (\u001b[1;33m<-\u001b[0m) like this will soon be deprecated:\n\n\u001b[1;36m231\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37m        value <- task |> await\u001b[0m\n              \u001b[1;33m^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\nYou should use a \u001b[1;33m!\u001b[0m for awaiting tasks or a \u001b[1;33m?\u001b[0m for trying results, and\nfunctions everywhere else.\n\n\n\u001b[1;36m── BACKPASSING DEPRECATED in ...O2HVVuPJFkwvvsfW6LojkLR20kTVE/InternalHttp.roc ─\u001b[0m\n\nBackpassing (\u001b[1;33m<-\u001b[0m) like this will soon be deprecated:\n\n\u001b[1;36m75\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37m    fmt <- Inspect.custom\u001b[0m\n         \u001b[1;33m^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\nYou should use a \u001b[1;33m!\u001b[0m for awaiting tasks or a \u001b[1;33m?\u001b[0m for trying results, and\nfunctions everywhere else.\n\n\u001b[1;36m────────────────────────────────────────────────────────────────────────────────\u001b[0m\n\n\u001b[1;32m0\u001b[0m errors and \u001b[1;33m3\u001b[0m warnings found\n.\n\nRunning tests…\n\n\u001b[36m────────────────────────────────────────────────────────────────────────────────\u001b[39m\n\u001b[1;36m── EXPECT FAILED in tests/partial-fail/partial-fail-test.roc ───────────────────\u001b[0m\n\nThis expectation failed:\n\n\u001b[1;36m6\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mexpect \"hello\" == \"good bye\"\u001b[0m\n    \u001b[1;31m^^^^^^^^^^^^^^^^^^^^^^^^^^^^\u001b[0m\n\n\n\u001b[1;31m1\u001b[0m failed and \u001b[1;32m1\u001b[0m passed."
 }

tests/partial-fail/partial-fail-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect 2 + 2 == 4
+expect "hello" == "good bye"
+
+main =
+    Task.ok {}

tests/success/success-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+import pf.Task exposing [Task]
+
+expect 2 + 2 == 4
+expect "four" == "four"
+
+main =
+    Task.ok {}

tests/syntax-error/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "error",
-  "message": "TODO: replace with correct output"
+  "message": "\u001b[1;36m── STATEMENT AFTER EXPRESSION in tests/syntax-error/syntax-error-test.roc ──────\u001b[0m\n\nI just finished parsing an expression with a series of definitions,\n\nand this line is indented as if it's intended to be part of that\nexpression:\n\n\u001b[1;36m1\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mapp [main] { pf: platform \"https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br\" }\u001b[0m\n\u001b[1;36m2\u001b[0m\u001b[1;36m│\u001b[0m\n\u001b[1;36m3\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37miMpOrT pf.Task exposing [Task]\u001b[0m\n\u001b[1;36m4\u001b[0m\u001b[1;36m│\u001b[0m\n\u001b[1;36m5\u001b[0m\u001b[1;36m│\u001b[0m  \u001b[37mexpect 2 + 2 == 5\u001b[0m\n    \u001b[1;31m^\u001b[0m\n\nHowever, I already saw the final expression in that series of\ndefinitions."
 }

tests/syntax-error/syntax-error-test.roc

@@ -0,0 +1,9 @@
+app [main] { pf: platform "https://github.com/roc-lang/basic-cli/releases/download/0.12.0/Lb8EgiejTUzbggO2HVVuPJFkwvvsfW6LojkLR20kTVE.tar.br" }
+
+iMpOrT pf.Task exposing [Task]
+
+expect 2 + 2 == 5
+expect "four" == "five"
+
+main =
+    Task.ok {}