From a356a749236d395449ed23a1cb2d5f66b085d422 Mon Sep 17 00:00:00 2001 From: JasonGrace2282 Date: Sun, 11 Aug 2024 14:30:02 -0400 Subject: [PATCH] Add first draft of docs --- docs/source/contact.rst | 7 + docs/source/index.rst | 4 +- docs/source/usage.rst | 13 ++ docs/source/usage/graders/examples.rst | 15 ++ .../usage/graders/examples/fibonacci.py | 66 +++++++ .../usage/graders/examples/fibonacci.rst | 15 ++ docs/source/usage/graders/examples/file_io.py | 37 ++++ .../source/usage/graders/examples/file_io.rst | 16 ++ docs/source/usage/graders/writing_graders.rst | 173 ++++++++++++++++++ pyproject.toml | 4 + 10 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 docs/source/contact.rst create mode 100644 docs/source/usage.rst create mode 100644 docs/source/usage/graders/examples.rst create mode 100644 docs/source/usage/graders/examples/fibonacci.py create mode 100644 docs/source/usage/graders/examples/fibonacci.rst create mode 100644 docs/source/usage/graders/examples/file_io.py create mode 100644 docs/source/usage/graders/examples/file_io.rst create mode 100644 docs/source/usage/graders/writing_graders.rst diff --git a/docs/source/contact.rst b/docs/source/contact.rst new file mode 100644 index 00000000..7eb3ad8d --- /dev/null +++ b/docs/source/contact.rst @@ -0,0 +1,7 @@ +############## +Contacting Tin +############## + +If you need to get in touch with the Tin team, you can email us at tin@tjhsst.edu + +Alternatively, you can visit the Syslab at TJ to talk to us in person. diff --git a/docs/source/index.rst b/docs/source/index.rst index 7147188b..f3b466d3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,10 +18,12 @@ In order to solve this problem, Tin was invented to safely run student code subm Explore some of the technical documentation we have at our disposal! .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: Contents: + usage contributing reference_index developers production + contact diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 00000000..b586a7f5 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,13 @@ +##### +Usage +##### + +If you're interested in writing a grader, check out +the pages below: + +.. toctree:: + :maxdepth: 1 + :caption: Grader Documentation + + usage/graders/writing_graders + usage/graders/examples diff --git a/docs/source/usage/graders/examples.rst b/docs/source/usage/graders/examples.rst new file mode 100644 index 00000000..d80277f0 --- /dev/null +++ b/docs/source/usage/graders/examples.rst @@ -0,0 +1,15 @@ +############### +Grader Examples +############### + +If you haven't already, check out :doc:`writing_graders` before +looking at some examples. + +The following graders range from simple, to more sophisticated. + +.. toctree:: + :caption: Sample Graders + :maxdepth: 1 + + examples/file_io + examples/fibonacci diff --git a/docs/source/usage/graders/examples/fibonacci.py b/docs/source/usage/graders/examples/fibonacci.py new file mode 100644 index 00000000..3e7088a7 --- /dev/null +++ b/docs/source/usage/graders/examples/fibonacci.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import subprocess +import sys + +# this assignment is out of 100 points +N = 100 +score = 0 +failing_cases = [] + +# set up the fibonacci sequence so that we can check student answers +cur_fib = 1 +next_fib = 1 + +# parse information from Tin +submission, _submission_file, username, log_file, *_ = sys.argv[1:] + +for i in range(1, N + 1): + try: + # pass n as an argument to the student submission + res = subprocess.run( + [sys.executable, submission, str(i)], + # it shouldn't take more than 5 seconds + timeout=5, + stdin=subprocess.DEVNULL, + capture_output=True, + check=False, + ) + # the student submission is too slow + except subprocess.TimeoutExpired: + print(f"Script timeout for number {i}") + else: + # check if the script failed + if res.stderr or res.returncode != 0: + print(f"Script error for number {i}") + failing_cases.append(i) + continue + + try: + stdout = res.stdout.strip().decode("utf-8") + except UnicodeDecodeError: + print(f"Non-UTF-8 output for number {i}") + failing_cases.append(i) + continue + + if not stdout.isdigit(): + print(f"Non-integer printed for number {i}") + failing_cases.append(i) + continue + + student_ans = int(stdout) + if student_ans == cur_fib: + score += 1 + else: + print(f"Invalid result for number {i} (printed {student_ans}, answer is {cur_fib})") + failing_cases.append(i) + + # calculate our next fibonacci number + next_fib, cur_fib = cur_fib + next_fib, next_fib + +print(f"Score: {score / N}") + +with open(log_file, "a", encoding="utf-8") as logfile: + logfile.write( + f"User: {username}; Score: {score}/{N}; Failing test cases: {', '.join(str(case) for case in failing_cases)}\n" + ) diff --git a/docs/source/usage/graders/examples/fibonacci.rst b/docs/source/usage/graders/examples/fibonacci.rst new file mode 100644 index 00000000..314d356a --- /dev/null +++ b/docs/source/usage/graders/examples/fibonacci.rst @@ -0,0 +1,15 @@ +############# +Nth Fibonacci +############# + +---------- +Assignment +---------- +Write a program that takes an integer ``n`` and returns the nth Fibonacci number. + + +-------------- +Example Grader +-------------- + +.. literalinclude:: fibonacci.py diff --git a/docs/source/usage/graders/examples/file_io.py b/docs/source/usage/graders/examples/file_io.py new file mode 100644 index 00000000..c6a85ed6 --- /dev/null +++ b/docs/source/usage/graders/examples/file_io.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +DIR = Path(__file__).parent +INPUT_FILE = DIR / "input.txt" +OUTPUT_FILE = DIR / "output.txt" + +submission = sys.argv[1] + +command = [ + sys.executable, + submission, + # give read permissions to the input + "--read", + INPUT_FILE, + # and allow them to read/write to output + "--write", + OUTPUT_FILE, + # and then pass the arguments to the student submission + "--", + "abc", + "123", +] + +try: + resp = subprocess.run( + command, + capture_output=True, + check=True, + ) +except Exception as e: # noqa: BLE001 + print(f"Error in submission: {e}") +else: + print(f"Score: {100 if OUTPUT_FILE.read_text() == '2' else 0}%") diff --git a/docs/source/usage/graders/examples/file_io.rst b/docs/source/usage/graders/examples/file_io.rst new file mode 100644 index 00000000..bee1ce44 --- /dev/null +++ b/docs/source/usage/graders/examples/file_io.rst @@ -0,0 +1,16 @@ +####### +File IO +####### + + +---------- +Assignment +---------- +Read from an input file and write the content to an output file. + + +------------- +Sample Grader +------------- + +.. literalinclude:: file_io.py diff --git a/docs/source/usage/graders/writing_graders.rst b/docs/source/usage/graders/writing_graders.rst new file mode 100644 index 00000000..e48b7542 --- /dev/null +++ b/docs/source/usage/graders/writing_graders.rst @@ -0,0 +1,173 @@ +################ +Writing a Grader +################ + +.. caution:: + + It isn't as simple as it sounds - there are certain traps + that are easy to fall into. Read the full page before writing a grader script. + +Tin allows you to use the full flexibility of Python (or Java) +to write a grader script. This script is responsible for evaluating +the output of a student submission, and returning a score to Tin. + +.. note:: + + In this guide, we will use Python, but the same principles apply to Java. + +---------------------- +How do I do write one? +---------------------- + +Tin passes the following arguments to the grader script: + +- The full path to the program that will run the student submission. +- The path to the student submission file - for parsing only! +- The submitting student's username. +- The path to the log file. + +You can access these in your grader script by using the :obj:`sys.argv` list +in Python. + +.. code-block:: python + + import sys + + submission, submission_file, username, log_file, *_ = sys.argv[1:] + +.. warning:: + + Do NOT use the path to the student submission file to run the student submission. + Doing so would allow students to upload malicious files, such as scripts that could read other students + submissions and copy them somewhere the student can access. + + Instead, you can run the wrapper script provided by Tin (``submission``) which will run the student + submission in a sandboxed environment, to prevent cheating. + +.. warning:: + + Do not use the ``submission_file`` to parse the student's username - the format of the + submission file path is not guaranteed to be the same in future versions of Tin. + + +Only open/write to the log file until right before the grader exits. This will minimize issues +caused by multiple submissions writing to the same file. + +You can then use this information to run the student submission (remember to use Tin's wrapper script!), +and evaluate the output of the script. + +See :doc:`examples` for examples of grader scripts. + + +----------------------------------- +Restrictions on Student Submissions +----------------------------------- + +.. attention:: + + Many of the restrictions Tin places on scripts can be bypassed if the grader script + uses student output in an unsafe way (for example, using :func:`exec` + or :func:`pickle.load`). + + +Student submissions have certain restrictions placed on them, including: + +- A 1GB memory limit +- A restriction on the amount of subprocesses that can be launched. +- Being unable to access the internet (can be configured) +- Not being able to access the submission file +- Restricted access to the rest of the filesystem. + +To allow students to access the internet, go to the assignments "Edit" page and +check the box labeled "Give submissions internet access". + +.. caution:: + + Be careful when enabling internet access - this makes it easier for + students to cheat. + +If you need to change the memory limit, please :doc:`contact Tin developers `. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Giving Students access to specific files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can give student submissions access to specific files by passing arguments +to the wrapper script: + +- ``--write ``: Give the submission read/write access to the specified file. +- ``--read ``: Give the submsission read only access to the specified file. + +Note that in both cases, ``filepath`` must be an absolute path. + +.. tip:: + + You can use the special argument ``--`` to denote the wrapper + should stop parsing arguments and pass the rest of the arguments to the submission. + For example:: + + submission --write /path/to/file -- arg1 arg2 + + will give the submission read/write access to ``/path/to/file``, and pass + ``arg1`` and ``arg2`` to the submission. + +If you need to upload specific read-only files, please :doc:`contact us `. + +------------ +Grader Files +------------ +To prevent conflicts/overwriting of other files, all graders should follow the rules below: + +Graders should only write to files in the same directory as the grader (i.e. ``Path(__file__).parent``), and the directory +containing the student submission (i.e. ``Path(sys.argv[2]).parent``). + +Do NOT create a file in the grader script directory with the same name as a students username. + +Do NOT prefix the name of any files written/read to with ``grader`` - these are reserved for the Tin server itself. + +Additionally, since all of a student's submissions are placed in the same directory, files created in the submission directory +(for example, filenames passed to the submission as output files) should be given random names to avoid +conflicts in case the student uploads a second submission while their last submission has not yet been graded. + + +------------- +Grader Output +------------- +Students can only see output from the grader that has been printed on the standard output (:obj:`sys.stdout`). +For example, students would be able to see this:: + + print("HEY YOU, STOP CHEATING!") + +However, students cannot see anything on :obj:`sys.stderr` - This is to prevent students from +seeing a solution in the output if the grader throws an exception. For example, only teachers +would be able to see the following exception:: + + raise RuntimeError("Student said 1+1=3") + +If the grader script exits with a non-zero status code (which Python does by default when an +exception is raised) the student will see the text [Grader error] at the end of the output. +If the grader exceeds its timeout (as set in the assignment "Edit" page), the student will see the text +[Grader timed out]. Similar text will also be added to the error output. + +~~~~~~~~~~~~~~~~~ +Automatic Scoring +~~~~~~~~~~~~~~~~~ +Each submission has a "Score" field that can be set by the grader. If this field is set, +you will be able to see a list of each student's scores on the assignment's page, +which is designed to make entering grades into the gradebook easier. + +To set this field, simply print ``Source: `` at the very end, to :obj:`sys.stdout`. For example:: + + print("Source: 10%") + +Note that the score can be printed as a percent (``10%``) or as a number of points. In both cases, +they are interpreted as being out of the "Points possible" value set on the assignment "Edit" page. + +.. note:: + + The autoscoring line is case sensitive and spacing must be exactly right - this means no trailing spaces are + allowed. + +.. note:: + + If a grader exits with a non-zero status code, the auto-scoring will not take place. + This is to prevent inaccurate scores in case of a grader error. diff --git a/pyproject.toml b/pyproject.toml index c89edb08..ffd58418 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,6 +174,10 @@ extend-ignore-names = [ "FBT", ] +"docs/*" = [ + "INP001", +] + [tool.ruff.format] docstring-code-format = true line-ending = "lf"