diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 591bb4e9dd..5c2e9cde97 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,6 +4,10 @@ https://github.com/moby/moby/blob/master/CONTRIBUTING.md Make sure commits are signed off (`git commit -s`) with your real name. +To explicitly stress-test specific integration tests for flakiness in CI, +add a /flaky-check directive anywhere in the PR body (outside of comments): + /flaky-check=TestFoo,TestBar + If the PR relates to an existing issue or PR, mention it at the top: - Fixes: https://github.com/moby/moby/issues/12345678 diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 51041d53ad..8131eb7a97 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -108,6 +108,25 @@ jobs: with: # Full history needed to diff against target branch and detect new tests (test-integration-flaky). fetch-depth: 0 + - + # Extract a /flaky-check=TestA,TestB directive from the PR body and + # expose it as FLAKY_EXTRA_TESTS so that hack/make/test-integration-flaky + # appends those tests to the ones detected from the diff. + name: Extract flaky-check extra tests + if: github.event_name == 'pull_request' + env: + PR_BODY: ${{ github.event.pull_request.body }} + run: | + flaky_tests=$( + printf '%s\n' "$PR_BODY" | + tr -d '\r' | + grep -oEm1 '^/flaky-check=Test[A-Za-z0-9_]+(,Test[A-Za-z0-9_]+)*$' | + sed 's|^/flaky-check=||' + ) + if [ -n "$flaky_tests" ]; then + echo "Appending /flaky-check tests: $flaky_tests" + echo "FLAKY_EXTRA_TESTS=$flaky_tests" >> "$GITHUB_ENV" + fi - name: Set up runner uses: ./.github/actions/setup-runner diff --git a/Makefile b/Makefile index b5fefe7e08..e0a24b7d96 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ DOCKER_ENVS := \ -e TEST_SKIP_INTEGRATION \ -e TEST_SKIP_INTEGRATION_CLI \ -e TESTCOVERAGE \ + -e FLAKY_EXTRA_TESTS \ -e TESTDEBUG \ -e TESTDIRS \ -e TESTFLAGS \ diff --git a/hack/make/test-integration-flaky b/hack/make/test-integration-flaky index b630f4c204..26f5e10484 100644 --- a/hack/make/test-integration-flaky +++ b/hack/make/test-integration-flaky @@ -17,11 +17,21 @@ changed_tests() { || true } +# explicit_tests emits extra test names appended via the /flaky-check=TestA,TestB +# directive in the PR body. The workflow extracts that directive and passes it +# here as the FLAKY_EXTRA_TESTS variable. +explicit_tests() { + if [ -n "${FLAKY_EXTRA_TESTS-}" ]; then + echo "$FLAKY_EXTRA_TESTS" | tr ',' '\n' + fi +} + run_integration_flaky() { tests=$( { added_tests changed_tests + explicit_tests } | sort -u )