diff --git a/pyproject.toml b/pyproject.toml index 9e2a35f..baf98d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ disable = [ "duplicate-code", "fixme", "import-error", + "logging-fstring-interpolation", "subprocess-run-check", "too-few-public-methods", "too-many-arguments", diff --git a/setup.cfg b/setup.cfg index 864d257..2328497 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,8 @@ name = lithium-reducer url = https://github.com/MozillaSecurity/lithium [options] +install_requires = + ffpuppet~=0.11.2 package_dir = = src packages = diff --git a/src/lithium/__init__.py b/src/lithium/__init__.py index 44c02ba..eaea4d9 100644 --- a/src/lithium/__init__.py +++ b/src/lithium/__init__.py @@ -1,4 +1,3 @@ -# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. diff --git a/src/lithium/interestingness/__init__.py b/src/lithium/interestingness/__init__.py index ee6f050..e69de29 100644 --- a/src/lithium/interestingness/__init__.py +++ b/src/lithium/interestingness/__init__.py @@ -1,7 +0,0 @@ -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. -"""lithium built-in interestingness tests""" - -from . import crashes, diff_test, hangs, outputs, repeat, timed_run, utils diff --git a/src/lithium/interestingness/crashes.py b/src/lithium/interestingness/crashes.py index 30da9dd..4b04903 100644 --- a/src/lithium/interestingness/crashes.py +++ b/src/lithium/interestingness/crashes.py @@ -1,4 +1,3 @@ -# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. @@ -10,13 +9,19 @@ """ import logging -from typing import List +import sys +from typing import List, Optional -from . import timed_run +from .timed_run import BaseParser, ExitStatus, timed_run +LOG = logging.getLogger(__name__) -def interesting(cli_args: List[str], temp_prefix: str) -> bool: - """Interesting if the binary causes a crash. (e.g. SIGKILL/SIGTERM/SIGTRAP etc.) + +def interesting( + cli_args: Optional[List[str]] = None, + temp_prefix: Optional[str] = None, +) -> bool: + """Interesting if the binary causes a crash. Args: cli_args: List of input arguments. @@ -25,20 +30,21 @@ def interesting(cli_args: List[str], temp_prefix: str) -> bool: Returns: True if binary crashes, False otherwise. """ - parser = timed_run.ArgumentParser( - prog="crashes", - usage="python -m lithium %(prog)s [options] binary [flags] testcase.ext", - ) + parser = BaseParser() args = parser.parse_args(cli_args) + if not args.cmd_with_flags: + parser.error("Must specify command to evaluate.") - log = logging.getLogger(__name__) - # Run the program with desired flags and look out for crashes. - runinfo = timed_run.timed_run(args.cmd_with_flags, args.timeout, temp_prefix) + run_info = timed_run(args.cmd_with_flags, args.timeout, temp_prefix) - time_str = f" ({runinfo.elapsedtime:.3f} seconds)" - if runinfo.sta == timed_run.CRASHED: - log.info("Exit status: " + runinfo.msg + time_str) + if run_info.status == ExitStatus.CRASH: + LOG.info(f"[Interesting] Crash detected ({run_info.elapsed:.3f}s)") return True - log.info("[Uninteresting] It didn't crash: " + runinfo.msg + time_str) + LOG.info(f"[Uninteresting] No crash detected ({run_info.elapsed:.3f}s)") return False + + +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + sys.exit(interesting()) diff --git a/src/lithium/interestingness/diff_test.py b/src/lithium/interestingness/diff_test.py index 9c21dad..8493a13 100644 --- a/src/lithium/interestingness/diff_test.py +++ b/src/lithium/interestingness/diff_test.py @@ -1,4 +1,3 @@ -# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. @@ -8,89 +7,118 @@ used to isolate and minimize differential behaviour test cases. Example: - python -m lithium diff_test -a "--fuzzing-safe" \ - -b "--fuzzing-safe --wasm-always-baseline" - -Example with autobisectjs, split into separate lines here for readability: - python -u -m funfuzz.autobisectjs.autobisectjs \ - -b "--enable-debug --enable-more-deterministic" -p testcase.js \ - -i diff_test -a "--fuzzing-safe --no-threads --ion-eager" \ - -b "--fuzzing-safe --no-threads --ion-eager --no-wasm-baseline" + python -m lithium diff_test \ + -a "--fuzzing-safe" \ + -b "--fuzzing-safe --wasm-always-baseline" \ + """ - -# This file came from nbp's GitHub PR #2 for adding new Lithium reduction strategies. -# https://github.com/MozillaSecurity/lithium/pull/2 - +import argparse import filecmp import logging -from typing import List +import sys +from typing import List, Optional, Union -from . import timed_run +from .timed_run import BaseParser, ExitStatus, timed_run +LOG = logging.getLogger(__name__) -def interesting(cli_args: List[str], temp_prefix: str) -> bool: - """Interesting if the binary shows a difference in output when different command - line arguments are passed in. + +def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse args Args: - cli_args: List of input arguments. - temp_prefix: Temporary directory prefix, e.g. tmp1/1 or tmp4/1 + argv: List of input arguments. Returns: - True if a difference in output appears, False otherwise. + Parsed arguments """ - parser = timed_run.ArgumentParser( + parser = BaseParser( prog="diff_test", - usage="python -m lithium %(prog)s [options] binary testcase.ext", + usage="python -m lithium.interestingness.diff " + "-a '--fuzzing-safe' -b='' binary testcase.js", ) parser.add_argument( "-a", - "--a-args", dest="a_args", help="Set of extra arguments given to first run.", + required=True, ) parser.add_argument( "-b", - "--b-args", dest="b_args", help="Set of extra arguments given to second run.", + required=True, ) - args = parser.parse_args(cli_args) - a_runinfo = timed_run.timed_run( - args.cmd_with_flags[:1] + args.a_args.split() + args.cmd_with_flags[1:], - args.timeout, - temp_prefix + "-a", - ) - b_runinfo = timed_run.timed_run( - args.cmd_with_flags[:1] + args.b_args.split() + args.cmd_with_flags[1:], - args.timeout, - temp_prefix + "-b", - ) - log = logging.getLogger(__name__) - time_str = ( - f"(1st Run: {a_runinfo.elapsedtime:.3f} seconds)" - f" (2nd Run: {b_runinfo.elapsedtime:.3f} seconds)" - ) + args = parser.parse_args(argv) + if not args.cmd_with_flags: + parser.error("Must specify command to evaluate.") - if timed_run.TIMED_OUT not in (a_runinfo.sta, b_runinfo.sta): - if a_runinfo.return_code != b_runinfo.return_code: - log.info( - "[Interesting] Different return code (%d, %d). %s", - a_runinfo.return_code, - b_runinfo.return_code, - time_str, - ) - return True - if not filecmp.cmp(a_runinfo.out, b_runinfo.out): - log.info("[Interesting] Different output. %s", time_str) - return True - if not filecmp.cmp(a_runinfo.err, b_runinfo.err): - log.info("[Interesting] Different error output. %s", time_str) + return args + + +def interesting( + cli_args: Optional[List[str]] = None, + temp_prefix: Optional[str] = None, +) -> bool: + """Check if there's a difference in output or return code with different args. + + Args: + cli_args: Input arguments. + temp_prefix: Temporary directory prefix, e.g. tmp1/1. + + Returns: + True if a difference in output appears, False otherwise. + """ + args = parse_args(cli_args) + + binary = args.cmd_with_flags[:1] + testcase = args.cmd_with_flags[1:] + + # Run with arguments set A + command_a = binary + args.a_args.split() + testcase + log_prefix_a = f"{temp_prefix}-a" if temp_prefix else None + a_run = timed_run(command_a, args.timeout, log_prefix_a) + if a_run.status == ExitStatus.TIMEOUT: + LOG.warning("Command A timed out!") + + # Run with arguments set B + command_b = binary + args.b_args.split() + testcase + log_prefix_b = f"{temp_prefix}-b" if temp_prefix else None + b_run = timed_run(command_b, args.timeout, log_prefix_b) + if b_run.status == ExitStatus.TIMEOUT: + LOG.warning("Command B timed out!") + + # Compare return codes + a_ret = a_run.return_code + b_ret = b_run.return_code + if a_ret != b_ret: + LOG.info(f"[Interesting] Different return codes: {a_ret} vs {b_ret}") + return True + + # Compare outputs + def cmp_out( + a_data: Union[str, bytes], + b_run: Union[str, bytes], + is_file: bool = False, + ) -> bool: + if is_file: + return not filecmp.cmp(a_data, b_run) + return a_data != b_run + + if temp_prefix: + if cmp_out(a_run.out, b_run.out, True) or cmp_out(a_run.err, b_run.err, True): + LOG.info("[Interesting] Differences in output detected") return True else: - log.info("[Uninteresting] At least one test timed out. %s", time_str) - return False + if cmp_out(a_run.out, b_run.out) or cmp_out(a_run.err, b_run.err): + LOG.info("[Interesting] Differences in output detected") + return True - log.info("[Uninteresting] Identical behaviour. %s", time_str) + LOG.info("[Uninteresting] No differences detected") return False + + +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + sys.exit(interesting()) diff --git a/src/lithium/interestingness/hangs.py b/src/lithium/interestingness/hangs.py index 07921c7..0e18e13 100644 --- a/src/lithium/interestingness/hangs.py +++ b/src/lithium/interestingness/hangs.py @@ -1,4 +1,3 @@ -# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. @@ -10,12 +9,18 @@ """ import logging -from typing import List +import sys +from typing import List, Optional -from . import timed_run +from .timed_run import BaseParser, ExitStatus, timed_run +LOG = logging.getLogger(__name__) -def interesting(cli_args: List[str], temp_prefix: str) -> bool: + +def interesting( + cli_args: Optional[List[str]] = None, + temp_prefix: Optional[str] = None, +) -> bool: """Interesting if the binary causes a hang. Args: @@ -25,18 +30,20 @@ def interesting(cli_args: List[str], temp_prefix: str) -> bool: Returns: True if binary causes a hang, False otherwise. """ - parser = timed_run.ArgumentParser( - prog="hangs", - usage="python -m lithium %(prog)s [options] binary [flags] testcase.ext", - ) + parser = BaseParser() args = parser.parse_args(cli_args) + if not args.cmd_with_flags: + parser.error("Must specify command to evaluate.") - log = logging.getLogger(__name__) - runinfo = timed_run.timed_run(args.cmd_with_flags, args.timeout, temp_prefix) - - if runinfo.sta == timed_run.TIMED_OUT: - log.info("Timed out after %.3f seconds", args.timeout) + run_info = timed_run(args.cmd_with_flags, args.timeout, temp_prefix) + if run_info.status == ExitStatus.TIMEOUT: + LOG.info(f"[Interesting] Timeout detected ({args.timeout:.3f}s)") return True - log.info("Exited in %.3f seconds", runinfo.elapsedtime) + LOG.info(f"[Uninteresting] Program exited ({run_info.elapsed:.3f}s)") return False + + +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + sys.exit(interesting()) diff --git a/src/lithium/interestingness/outputs.py b/src/lithium/interestingness/outputs.py index ce4307b..71b9857 100644 --- a/src/lithium/interestingness/outputs.py +++ b/src/lithium/interestingness/outputs.py @@ -1,7 +1,7 @@ -# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. + """Lithium's "outputs" interestingness test to assess whether an intended message shows up. @@ -10,14 +10,37 @@ """ import logging -import os +import re +import sys from pathlib import Path -from typing import List, Union +from typing import List, Optional, Union + +from . import utils +from .timed_run import BaseParser, timed_run + +LOG = logging.getLogger(__name__) + + +def file_contains(path: Union[Path, str], is_regex: bool, search: str) -> bool: + """Determine if string is present in a file. + + Args: + path: + is_regex: + search: -from . import timed_run, utils + Returns: + + """ + if is_regex: + return utils.file_contains_regex(path, search.encode())[0] + return utils.file_contains_str(path, search.encode()) -def interesting(cli_args: List[str], temp_prefix: str) -> bool: +def interesting( + cli_args: Optional[List[str]] = None, + temp_prefix: Optional[str] = None, +) -> bool: """Interesting if the binary causes an intended message to show up. (e.g. on stdout/stderr) @@ -28,38 +51,50 @@ def interesting(cli_args: List[str], temp_prefix: str) -> bool: Returns: True if the intended message shows up, False otherwise. """ - parser = timed_run.ArgumentParser( - prog="outputs", - usage="python -m lithium %(prog)s [options] output_message binary [flags] " - "testcase.ext", + parser = BaseParser() + parser.add_argument( + "-s", + "--search", + help="String to search for.", + required=True, ) parser.add_argument( "-r", "--regex", action="store_true", default=False, - help="Allow search for regular expressions instead of strings.", + help="Treat string as a regular expression", ) args = parser.parse_args(cli_args) + if not args.cmd_with_flags: + parser.error("Must specify command to evaluate.") - log = logging.getLogger(__name__) + run_info = timed_run(args.cmd_with_flags, args.timeout, temp_prefix) - search_for = args.cmd_with_flags[0] - if not isinstance(search_for, bytes): - search_for = os.fsencode(search_for) + if temp_prefix is None: + outputs = (run_info.out, run_info.err) + for data in outputs: + if (args.regex and re.match(args.search, data, flags=re.MULTILINE)) or ( + args.search in data + ): + LOG.info("[Interesting] Match detected!") + return True - # Run the program with desired flags and search stdout and stderr for intended - # message - runinfo = timed_run.timed_run(args.cmd_with_flags[1:], args.timeout, temp_prefix) - - def file_contains(path: Union[Path, str]) -> bool: - if args.regex: - return utils.file_contains_regex(path, search_for)[0] - return utils.file_contains_str(path, search_for) + LOG.info("[Uninteresting] No match detected!") + return False result = any( - file_contains(temp_prefix + suffix) for suffix in ("-out.txt", "-err.txt") + file_contains(f"{temp_prefix}{suffix}", args.regex, args.search) + for suffix in ("-out.txt", "-err.txt") ) + if result: + LOG.info("[Interesting] Match detected!") + return True + + LOG.info("[Uninteresting] No match detected!") + return False + - log.info("Exit status: %s (%.3f seconds)", runinfo.msg, runinfo.elapsedtime) - return result +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + sys.exit(interesting()) diff --git a/src/lithium/interestingness/timed_run.py b/src/lithium/interestingness/timed_run.py index 773e81b..fda77d0 100644 --- a/src/lithium/interestingness/timed_run.py +++ b/src/lithium/interestingness/timed_run.py @@ -1,12 +1,11 @@ -# # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -"""Run a subprocess with timeout -""" - +"""Run a subprocess with timeout""" import argparse -import collections +import enum +import logging +import os import platform import signal import subprocess @@ -15,17 +14,14 @@ from pathlib import Path from typing import BinaryIO, Callable, Dict, List, Optional, Union -(CRASHED, TIMED_OUT, NORMAL, ABNORMAL, NONE) = range(5) +from ffpuppet import SanitizerOptions +LOG = logging.getLogger(__name__) -# Define struct that contains data from a process that has already ended. -RunData = collections.namedtuple( - "RunData", - "sta, return_code, msg, elapsedtime, killed, out, err, pid", -) +ERROR_CODE = 77 -class ArgumentParser(argparse.ArgumentParser): +class BaseParser(argparse.ArgumentParser): """Argument parser with `timeout` and `cmd_with_args`""" def __init__(self, *args, **kwds) -> None: # type: ignore @@ -41,9 +37,77 @@ def __init__(self, *args, **kwds) -> None: # type: ignore self.add_argument("cmd_with_flags", nargs=argparse.REMAINDER) -def get_signal_name(signum: int, default: str = "Unknown signal") -> str: - """Stringify a signal number. The result will be something like "SIGSEGV", - or from Python 3.8, "Segmentation fault". +class ExitStatus(enum.IntEnum): + """Enum for recording exit status""" + + NORMAL = 1 + ABNORMAL = 2 + CRASH = 3 + TIMEOUT = 4 + + +class RunData: + """Class for storing run data""" + + def __init__( + self, + pid: int, + status: ExitStatus, + return_code: Union[int, None], + message: str, + elapsed: float, + out: Union[bytes, str], + err: Union[bytes, str], + ): + self.pid = pid + self.status = status + self.return_code = return_code + self.message = message + self.elapsed = elapsed + self.out = out + self.err = err + + +def _configure_sanitizers(orig_env: Dict[str, str]) -> Dict[str, str]: + """Copy environment and update default values in *SAN_OPTIONS entries. + + Args: + orig_env: Current environment. + + Returns: + Environment with *SAN_OPTIONS defaults set. + """ + env: Dict[str, str] = dict(orig_env) + # https://github.com/google/sanitizers/wiki/SanitizerCommonFlags + common_flags = [ + ("abort_on_error", "false"), + ("allocator_may_return_null", "true"), + ("disable_coredump", "true"), + ("exitcode", str(ERROR_CODE)), # use unique exitcode + ("handle_abort", "true"), # if true, abort_on_error=false to prevent hangs + ("handle_sigbus", "true"), # set to be safe + ("handle_sigfpe", "true"), # set to be safe + ("handle_sigill", "true"), # set to be safe + ("symbolize", "true"), + ] + + sanitizer_env_variables = ( + "ASAN_OPTIONS", + "UBSAN_OPTIONS", + "LSAN_OPTIONS", + "TSAN_OPTIONS", + ) + for sanitizer in sanitizer_env_variables: + config = SanitizerOptions(env.get(sanitizer)) + for flag in common_flags: + config.add(*flag) + env[sanitizer] = str(config) + + return env + + +def _get_signal_name(signum: int, default: str = "Unknown signal") -> str: + """Stringify a signal number Args: signum: Signal number to lookup @@ -80,24 +144,15 @@ def timed_run( preexec_fn: called in child process after fork, prior to exec Raises: - TypeError: Raises if input parameters are not of the desired types - (e.g. cmd_with_args should be a list) OSError: Raises if timed_run is attempted to be used with gdb Returns: - A rundata instance containing run information + A RunData instance containing run information. """ - if not isinstance(cmd_with_args, list): - raise TypeError("cmd_with_args should be a list (of strings).") - if not isinstance(timeout, int): - raise TypeError("timeout should be an int.") - if log_prefix is not None and not isinstance(log_prefix, str): - raise TypeError("log_prefix should be a string.") - if preexec_fn is not None and not hasattr(preexec_fn, "__call__"): - raise TypeError("preexec_fn should be callable.") - - prog = Path(cmd_with_args[0]).expanduser() - cmd_with_args[0] = str(prog) + if len(cmd_with_args) == 0: + raise ValueError("Command not specified!") + + prog = Path(cmd_with_args[0]).resolve() if prog.stem == "gdb": raise OSError( @@ -105,18 +160,18 @@ def timed_run( "kill gdb but leave the process within gdb still running" ) - sta = NONE - msg = "" - - child_stderr: Union[int, BinaryIO] = subprocess.PIPE - child_stdout: Union[int, BinaryIO] = subprocess.PIPE + status = None + env = _configure_sanitizers(os.environ.copy() if env is None else env) + child_stderr: Union[BinaryIO, int] = subprocess.PIPE + child_stdout: Union[BinaryIO, int] = subprocess.PIPE if log_prefix is not None: # pylint: disable=consider-using-with - child_stdout = open(log_prefix + "-out.txt", "wb") - child_stderr = open(log_prefix + "-err.txt", "wb") + child_stdout = open(f"{log_prefix}-out.txt", "wb") + child_stderr = open(f"{log_prefix}-err.txt", "wb") start_time = time.time() # pylint: disable=consider-using-with,subprocess-popen-preexec-fn + LOG.info(f"Running: {' '.join(cmd_with_args)}") child = subprocess.Popen( cmd_with_args, env=env, @@ -132,12 +187,9 @@ def timed_run( except subprocess.TimeoutExpired: child.kill() stdout, stderr = child.communicate() - sta = TIMED_OUT + status = ExitStatus.TIMEOUT except Exception as exc: # pylint: disable=broad-except - print("Tried to run:") - print(f" {cmd_with_args!r}") - print("but got this error:") - print(f" {exc}") + LOG.error(exc) sys.exit(2) finally: if isinstance(child_stderr, BinaryIO) and isinstance(child_stdout, BinaryIO): @@ -145,33 +197,31 @@ def timed_run( child_stderr.close() elapsed_time = time.time() - start_time - if sta == TIMED_OUT: - msg = "TIMED OUT" + if status == ExitStatus.TIMEOUT: + message = "TIMED OUT" elif child.returncode == 0: - msg = "NORMAL" - sta = NORMAL - elif 0 < child.returncode < 0x80000000: - msg = "ABNORMAL exit code " + str(child.returncode) - sta = ABNORMAL + message = "NORMAL" + status = ExitStatus.NORMAL + elif child.returncode != ERROR_CODE and 0 < child.returncode < 0x80000000: + message = f"ABNORMAL exit code {child.returncode}" + status = ExitStatus.ABNORMAL else: - # return_code < 0 (or > 0x80000000 in Windows) - # The program was terminated by a signal, which usually indicates a crash. + # The program was terminated by a signal or by the sanitizer (ERROR_CODE) # Mac/Linux only! - # XXX: this doesn't work on Windows if child.returncode < 0: - signum = -child.returncode + signum = abs(child.returncode) + message = f"CRASHED with {_get_signal_name(signum)}" else: - signum = child.returncode - msg = f"CRASHED signal {signum} ({get_signal_name(signum)})" - sta = CRASHED + message = "CRASHED" + + status = ExitStatus.CRASH return RunData( - sta, - child.returncode if sta != TIMED_OUT else None, - msg, - elapsed_time, - sta == TIMED_OUT, - log_prefix + "-out.txt" if log_prefix is not None else stdout, - log_prefix + "-err.txt" if log_prefix is not None else stderr, child.pid, + status, + child.returncode if status != ExitStatus.TIMEOUT else None, + message, + elapsed_time, + stdout if log_prefix is None else f"{log_prefix}-out.txt", + stderr if log_prefix is None else f"{log_prefix}-err.txt", ) diff --git a/src/lithium/interestingness/utils.py b/src/lithium/interestingness/utils.py index 2e08ba6..c7c48d4 100644 --- a/src/lithium/interestingness/utils.py +++ b/src/lithium/interestingness/utils.py @@ -18,7 +18,9 @@ def file_contains_str( - input_file: Union[Path, str], regex: bytes, verbose: bool = True + input_file: Union[Path, str], + regex: bytes, + verbose: bool = True, ) -> bool: """Helper function to check if file contains a given string @@ -47,7 +49,9 @@ def file_contains_str( def file_contains_regex( - input_file: Union[Path, str], regex: bytes, verbose: bool = True + input_file: Union[Path, str], + regex: bytes, + verbose: bool = True, ) -> Tuple[bool, bytes]: """e.g. python -m lithium crashesat --timeout=30 \ --regex '^#0\\s*0x.* in\\s*.*(?:\\n|\\r\\n?)#1\\s*' \ diff --git a/tests/test_interestingness.py b/tests/test_interestingness.py index c87c45b..9f99814 100644 --- a/tests/test_interestingness.py +++ b/tests/test_interestingness.py @@ -176,10 +176,10 @@ def test_diff_test_1() -> None: "--strategy", "check-only", "diff_test", - "--a-args", - "flags_one_a flags_one_b", - "--b-args", - "flags_two", + "-a", + "'--fuzzing-safe'", + "-b", + "'--fuzzing-safe --ion-offthread-compile=off'", ] + LS_CMD + ["temp.js"] @@ -220,7 +220,9 @@ def test_outputs_true() -> None: # test that `ls temp.js` contains "temp.js" result = lith.main( - ["--strategy", "check-only", "outputs", "temp.js"] + LS_CMD + ["temp.js"] + ["--strategy", "check-only", "outputs", "--search", "temp.js"] + + LS_CMD + + ["temp.js"] ) assert result == 0 assert lith.test_count == 1 @@ -232,7 +234,9 @@ def test_outputs_false() -> None: # test that `ls temp.js` does not contain "blah" result = lith.main( - ["--strategy", "check-only", "outputs", "blah"] + LS_CMD + ["temp.js"] + ["--strategy", "check-only", "outputs", "--search", "blah"] + + LS_CMD + + ["temp.js"] ) assert result == 1 assert lith.test_count == 1 @@ -253,6 +257,7 @@ def test_outputs_timeout() -> None: "outputs", "--timeout", "1", + "--search", "blah", ] + SLEEP_CMD @@ -270,7 +275,7 @@ def test_outputs_regex() -> None: # test that regex matches work too result = lith.main( - ["--strategy", "check-only", "outputs", "--regex", r"^.*js\s?$"] + ["--strategy", "check-only", "outputs", "--search", r"^.*js\s?$", "--regex"] + LS_CMD + ["temp.js"] ) @@ -287,7 +292,7 @@ def test_repeat_0() -> None: # Check for a known string result = lith.main( ["--strategy", "check-only"] - + ["repeat", "5", "outputs", "hello"] + + ["repeat", "5", "outputs", "--search", "hello"] + CAT_CMD + ["temp.js"] ) @@ -306,7 +311,7 @@ def test_repeat_1(caplog) -> None: caplog.clear() result = lith.main( ["--strategy", "check-only"] - + ["repeat", "5", "outputs", "notfound"] + + ["repeat", "5", "outputs", "--search", "notfound"] + CAT_CMD + ["temp.js"] ) @@ -335,7 +340,7 @@ def test_repeat_2() -> None: tempf1a.write("num0") result = lith.main( ["--strategy", "check-only"] - + ["repeat", "1", "outputs", "--timeout=9", "numREPEATNUM"] + + ["repeat", "1", "outputs", "--timeout=9", "--search", "numREPEATNUM"] + CAT_CMD + ["temp.js"] ) @@ -352,7 +357,7 @@ def test_repeat_3() -> None: tempf1b.write("num2") result = lith.main( ["--strategy", "check-only"] - + ["repeat", "1", "outputs", "--timeout=9", "numREPEATNUM"] + + ["repeat", "1", "outputs", "--timeout=9", "--search", "numREPEATNUM"] + CAT_CMD + ["temp.js"] ) @@ -369,7 +374,7 @@ def test_repeat_4() -> None: tempf2a.write("num0") result = lith.main( ["--strategy", "check-only"] - + ["repeat", "2", "outputs", "--timeout=9", "numREPEATNUM"] + + ["repeat", "2", "outputs", "--timeout=9", "--search", "numREPEATNUM"] + CAT_CMD + ["temp.js"] ) @@ -386,7 +391,7 @@ def test_repeat_5() -> None: tempf2b.write("num3") result = lith.main( ["--strategy", "check-only"] - + ["repeat", "2", "outputs", "--timeout=9", "numREPEATNUM"] + + ["repeat", "2", "outputs", "--timeout=9", "--search", "numREPEATNUM"] + CAT_CMD + ["temp.js"] ) @@ -417,6 +422,7 @@ def test_interestingness_outputs_multiline(capsys, pattern, expected) -> None: result = lith.main( [ "outputs", + "--search", pattern, ] + CAT_CMD