feat: ignore flaky tests (#238)

Added a new optional parameter ignore_flaky_tests which is "false" by default (current behavior) When set to "true" : flaky tests are considered as success and will generate a successful report --------- Co-authored-by: Ghais Zaher <[email protected]>
ScaCap · Jan 8, 2025 · 1a128e4 · 1a128e4
1 parent 6966497
commit 1a128e4
Show file tree

Hide file tree

Showing 10 changed files with 1,210 additions and 353 deletions.
diff --git a/README.md b/README.md
@@ -33,6 +33,10 @@ Optional. Check will fail if there are test failures. The default is `false`.
 
 Optional. Check will fail if no tests were found. The default is `true`.
 
+### `ignore_flaky_tests`
+
+Optional. Set to `true` to consider flaky tests as success. The default is `false`.
+
 ### `skip_publishing`
 
 Optional. Skip the test report publishing (check run creation). The default is `false`.

diff --git a/action.js b/action.js
@@ -15,11 +15,12 @@ const action = async () => {
     const commit = core.getInput('commit');
     const failOnFailedTests = core.getInput('fail_on_test_failures') === 'true';
     const failIfNoTests = core.getInput('fail_if_no_tests') === 'true';
+    const ignoreFlakyTests = core.getInput('ignore_flaky_tests') === 'true';
     const skipPublishing = core.getInput('skip_publishing') === 'true';
     const isFilenameInStackTrace = core.getInput('file_name_in_stack_trace') === 'true';
     const githubBaseUrl = core.getInput('github_base_url');
 
-    let { count, skipped, annotations } = await parseTestReports(reportPaths, isFilenameInStackTrace);
+    let { count, skipped, annotations } = await parseTestReports(reportPaths, isFilenameInStackTrace, ignoreFlakyTests);
     const foundResults = count > 0 || skipped > 0;
     const conclusion =
         (foundResults && annotations.length === 0) || (!foundResults && !failIfNoTests)

diff --git a/action.test.fixtures.js b/action.test.fixtures.js
@@ -4,7 +4,7 @@ const finishedWithFailures = {
     status: 'completed',
     conclusion: 'failure',
     output: {
-        title: '19 tests run, 1 skipped, 12 failed.',
+        title: '20 tests run, 1 skipped, 13 failed.',
         summary: '',
         annotations: [
             {
@@ -112,6 +112,17 @@ const finishedWithFailures = {
                 raw_details:
                     "java.lang.AssertionError: \n\nExpected: \"Good Twin\"\n     but: was \"Evil Twin\"\n\tat action.surefire.report.twin.second.TwinTest.should_always_fail(TwinTest.java:13)"
             },
+            {
+                path: 'integration-tests/maven/flakes/src/test/java/action/surefire/report/calc/AllOkWithFlakesTest.java',
+                start_line: 1,
+                end_line: 1,
+                start_column: 0,
+                end_column: 0,
+                annotation_level: 'failure',
+                title: 'AllOkWithFlakesTest.firstTryFailSecondTrySuccess',
+                message: 'firstTryFailSecondTrySuccess',
+                raw_details: ""
+            },
             {
                 path: 'integration-tests/maven/utils/src/test/java/action/surefire/report/calc/CalcUtilsTest.kt',
                 start_line: 27,

diff --git a/action.test.js b/action.test.js
@@ -107,6 +107,24 @@ describe('action should work', () => {
         expect(failed).toBeNull();
     });
 
+    it('should send all ok if tests were flaky and ignore_flaky_test is true', async () => {
+        inputs.report_paths = '**/surefire-reports/TEST-*AllOkWithFlakesTest.xml';
+        inputs.ignore_flaky_tests = 'true';
+        let request = null;
+        const scope = nock('https://api.github.com')
+            .post('/repos/scacap/action-surefire-report/check-runs', body => {
+                request = body;
+                return body;
+            })
+            .reply(200, {});
+        await action();
+        scope.done();
+
+        expect(request).toStrictEqual(finishedSuccess);
+        expect(outputs).toHaveProperty('conclusion', 'success');
+        expect(failed).toBeNull();
+    });
+
     it('should send failure if no test results were found', async () => {
         inputs.report_paths = '**/xxx/*.xml';
         let request = null;
@@ -191,7 +209,7 @@ describe('action should work', () => {
             await action();
             scope.done();
 
-            expect(failed).toBe('There were 12 failed tests');
+            expect(failed).toBe('There were 13 failed tests');
         });
     });
 

diff --git a/action.yml b/action.yml
@@ -31,6 +31,10 @@ inputs:
     description: 'fail run if there were no test results found'
     required: false
     default: 'true'
+  ignore_flaky_tests:
+    description: 'consider flaky tests as success'
+    required: false
+    default: 'false'
   skip_publishing:
     description: 'skip test report publishing'
     required: false