diff --git a/run-vmtest/merge_test_lists.py b/run-vmtest/merge_test_lists.py deleted file mode 100755 index 7505187..0000000 --- a/run-vmtest/merge_test_lists.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python3 - -import sys - -def clean_line(line: str) -> str: - line = line.split('#')[0] # remove comment - line = line.strip() # strip whitespace - return line - -def read_clean_and_sort_input() -> list[str]: - input = [] - for line in sys.stdin: - line = clean_line(line) - if len(line) == 0: - continue - input.append(line) - input.sort() - return input - -def dedup_subtests(lines: list[str]): - i = 0 - j = 1 - while j < len(lines): - l1 = lines[i] - l2 = lines[j] - # for a pair of "foo_test" and "foo_test/subtest" - # remove a subtest from the list - if l1 == l2 or ('/' not in l1) and ('/' in l2) and l2.startswith(l1): - # print(f"removing '{l2}' because '{l1}' is in the list") - lines.remove(l2) - else: - i += 1 - j += 1 - -if __name__ == '__main__': - lines = read_clean_and_sort_input() - dedup_subtests(lines) - for line in lines: - print(line) - - diff --git a/run-vmtest/normalize_bpf_test_names.py b/run-vmtest/normalize_bpf_test_names.py new file mode 100755 index 0000000..b8a8828 --- /dev/null +++ b/run-vmtest/normalize_bpf_test_names.py @@ -0,0 +1,70 @@ +#!/usr/bin/python3 + +# This script normalizes a list of selftests/bpf test names. These +# lists have the following format: +# * each line indicates a test name +# * a line can contain an end-line comment, starting with # +# +# The test names may contain spaces, commas, and potentially other characters. +# +# The purpose of this script is to take in a composite allow/denylist +# (usually produced as an append of multiple different lists) and +# transform it into one clean deduplicated list. +# +# In addition to dedup of tests by exact match, subtests are taken +# into account. For example, one source denylist may contain a test +# "a_test/subtest2", and another may contain simply "a_test". In such +# case "a_test" indicates that no subtest of "a_test" should run, and +# so "a_test/subtest2" shouldn't be in the list. +# +# The result is printed to stdout + +import sys + +def clean_line(line: str) -> str: + line = line.split('#')[0] # remove comment + line = line.strip() # strip whitespace + return line + +def read_clean_and_sort_input(file) -> list[str]: + input = [] + for line in file: + line = clean_line(line) + if len(line) == 0: + continue + input.append(line) + input.sort() + return input + +# Deduplicate subtests and yield the next unique test name +def next_test(lines: list[str]): + + if not lines: + return + + prev = lines[0] + + def is_subtest(line: str) -> bool: + return ('/' in line) and ('/' not in prev) and line.startswith(prev) + + yield lines[0] + for line in lines[1:]: + if prev == line or is_subtest(line): + continue + yield line + prev = line + + +if __name__ == '__main__': + + if len(sys.argv) != 2: + print("Usage: merge_test_lists.py ", file=sys.stderr) + sys.exit(1) + + lines = [] + with open(sys.argv[1]) as file: + lines = read_clean_and_sort_input(file) + + for line in next_test(lines): + print(line) + diff --git a/run-vmtest/prepare-bpf-selftests.sh b/run-vmtest/prepare-bpf-selftests.sh index ed027eb..89ae79a 100755 --- a/run-vmtest/prepare-bpf-selftests.sh +++ b/run-vmtest/prepare-bpf-selftests.sh @@ -6,18 +6,20 @@ function merge_test_lists_into() { local out="$1" shift local files=("$@") - echo -n > "$out" - # first, append all the input lists into one + local list=$(mktemp) + echo -n > "$list" + # append all the source lists into one for file in "${files[@]}"; do if [[ -f "$file" ]]; then - echo "cat $file >> $out" - cat "$file" >> "$out" + echo "Include $file" + cat "$file" >> "$list" fi done - # then merge the list of test names - cat "$out" | python3 "$(dirname $0)/merge_test_lists.py" > "$out" + # then normalize the list of test names + $GITHUB_ACTION_PATH/normalize_bpf_test_names.py "$list" > "$out" + rm "$list" } # Read arrays from pipe-separated strings diff --git a/run-vmtest/tests/merge_test_lists/expected-output.txt b/run-vmtest/tests/normalize_bpf_test_names/expected-output.txt similarity index 100% rename from run-vmtest/tests/merge_test_lists/expected-output.txt rename to run-vmtest/tests/normalize_bpf_test_names/expected-output.txt diff --git a/run-vmtest/tests/merge_test_lists/input.txt b/run-vmtest/tests/normalize_bpf_test_names/input.txt similarity index 100% rename from run-vmtest/tests/merge_test_lists/input.txt rename to run-vmtest/tests/normalize_bpf_test_names/input.txt diff --git a/run-vmtest/tests/merge_test_lists/run-test.sh b/run-vmtest/tests/normalize_bpf_test_names/run-test.sh similarity index 62% rename from run-vmtest/tests/merge_test_lists/run-test.sh rename to run-vmtest/tests/normalize_bpf_test_names/run-test.sh index 84aa342..fd52d35 100755 --- a/run-vmtest/tests/merge_test_lists/run-test.sh +++ b/run-vmtest/tests/normalize_bpf_test_names/run-test.sh @@ -5,6 +5,6 @@ set -euo pipefail GITHUB_ACTION_PATH=$(realpath ../..) rm -f output.txt -cat input.txt | $GITHUB_ACTION_PATH/merge_test_lists.py 2>&1 > output.txt +$GITHUB_ACTION_PATH/normalize_bpf_test_names.py input.txt 2>&1 > output.txt diff expected-output.txt output.txt