fix(ci): cool down main workflow fanout

2026-08-07 10:34:44 +00:00 · 2026-06-20 17:32:14 +08:00
parent 7679872ddf
commit 4f8fd48ea7
6 changed files with 115 additions and 24 deletions
--- a/.github/workflows/clawsweeper-dispatch.yml
+++ b/.github/workflows/clawsweeper-dispatch.yml
@@ -18,8 +18,8 @@ permissions:
  contents: read

 concurrency:
-  group: clawsweeper-dispatch-${{ github.repository }}-${{ github.event.issue.number || github.event.pull_request.number || github.run_id }}
-  cancel-in-progress: ${{ github.event.action == 'edited' || github.event.action == 'synchronize' || github.event.action == 'ready_for_review' }}
+  group: ${{ github.event_name == 'push' && format('clawsweeper-dispatch-{0}-{1}', github.repository, github.ref) || format('clawsweeper-dispatch-{0}-{1}', github.repository, github.event.issue.number || github.event.pull_request.number || github.run_id) }}
+  cancel-in-progress: ${{ github.event_name == 'push' || github.event.action == 'edited' || github.event.action == 'synchronize' || github.event.action == 'ready_for_review' }}

 jobs:
  dispatch:
@@ -41,6 +41,34 @@ jobs:
        if: ${{ github.event.action == 'labeled' || github.event.action == 'unlabeled' }}
        run: sleep 20

+      - name: Debounce main push dispatch
+        if: ${{ github.event_name == 'push' }}
+        run: sleep 45
+
+      - name: Install GitHub API backoff helper
+        run: |
+          cat > "$RUNNER_TEMP/github-api-backoff.sh" <<'BASH'
+          gh_api_with_retry() {
+            local attempt output status lower_output
+            for attempt in 1 2 3 4 5; do
+              if output="$(gh api "$@" 2>&1)"; then
+                printf '%s\n' "$output"
+                return 0
+              fi
+              status=$?
+              lower_output="${output,,}"
+              if [[ "$lower_output" != *"rate limit"* && "$output" != *"HTTP 429"* ]]; then
+                printf '%s\n' "$output" >&2
+                return "$status"
+              fi
+              echo "::warning::GitHub API throttled ClawSweeper dispatch on attempt ${attempt}; retrying after backoff." >&2
+              sleep $((attempt * attempt * 5))
+            done
+            printf '%s\n' "$output" >&2
+            return "$status"
+          }
+          BASH
+
      - name: Create ClawSweeper dispatch token
        id: token
        if: ${{ env.HAS_CLAWSWEEPER_APP_PRIVATE_KEY == 'true' }}
@@ -77,6 +105,7 @@ jobs:
            echo "::notice::Skipping GitHub activity dispatch because no ClawSweeper app token is configured."
            exit 0
          fi
+          . "$RUNNER_TEMP/github-api-backoff.sh"
          activity="$(jq -c \
            --arg target_repo "$TARGET_REPO" \
            --arg event_name "$SOURCE_EVENT" \
@@ -143,7 +172,7 @@ jobs:
            ' "$GITHUB_EVENT_PATH")"
          payload="$(jq -nc --argjson activity "$activity" \
            '{event_type:"github_activity",client_payload:{activity:$activity}}')"
-          if gh api repos/openclaw/clawsweeper/dispatches \
+          if gh_api_with_retry repos/openclaw/clawsweeper/dispatches \
            --method POST \
            --input - <<< "$payload"; then
            echo "Dispatched GitHub activity to ClawSweeper."
@@ -165,6 +194,7 @@ jobs:
            echo "::notice::Skipping ClawSweeper dispatch because no ClawSweeper app token is configured. Not falling back to a maintainer token."
            exit 0
          fi
+          . "$RUNNER_TEMP/github-api-backoff.sh"
          payload="$(jq -nc \
            --arg target_repo "$TARGET_REPO" \
            --argjson item_number "$ITEM_NUMBER" \
@@ -173,7 +203,7 @@ jobs:
            --arg source_action "$SOURCE_ACTION" \
            --argjson supersedes_in_progress "$SUPERSEDES_IN_PROGRESS" \
            '{event_type:"clawsweeper_item",client_payload:{target_repo:$target_repo,item_number:$item_number,item_kind:$item_kind,source_event:$source_event,source_action:$source_action,supersedes_in_progress:$supersedes_in_progress}}')"
-          if gh api repos/openclaw/clawsweeper/dispatches \
+          if gh_api_with_retry repos/openclaw/clawsweeper/dispatches \
            --method POST \
            --input - <<< "$payload"; then
            echo "Dispatched ClawSweeper review."
@@ -198,6 +228,7 @@ jobs:
            echo "::notice::Skipping ClawSweeper comment dispatch because no ClawSweeper app token is configured."
            exit 0
          fi
+          . "$RUNNER_TEMP/github-api-backoff.sh"
          body_file="$RUNNER_TEMP/clawsweeper-comment-body.txt"
          printf '%s\n' "$COMMENT_BODY" > "$body_file"
          if ! grep -Eiq '(^|[[:space:]])@(clawsweeper|openclaw-clawsweeper)\b(\[bot\])?|(^|[[:space:]])/(clawsweeper|review|automerge|autoclose)\b' "$body_file"; then
@@ -206,7 +237,7 @@ jobs:
          fi
          if [ -n "$TARGET_TOKEN" ]; then
            err="$(mktemp)"
-            if GH_TOKEN="$TARGET_TOKEN" gh api -X POST \
+            if GH_TOKEN="$TARGET_TOKEN" gh_api_with_retry -X POST \
              -H "Accept: application/vnd.github+json" \
              "repos/$TARGET_REPO/issues/comments/$COMMENT_ID/reactions" \
              -f content="eyes" 2>"$err" >/dev/null; then
@@ -233,7 +264,7 @@ jobs:
                  "Command router queued. I will update this comment with the next step.")"
                status_payload="$(jq -nc --arg body "$status_body" '{body:$body}')"
                status_err="$(mktemp)"
-                if status_response="$(GH_TOKEN="$TARGET_TOKEN" gh api \
+                if status_response="$(GH_TOKEN="$TARGET_TOKEN" gh_api_with_retry \
                  "repos/$TARGET_REPO/issues/$ITEM_NUMBER/comments" \
                  --method POST \
                  --input - <<< "$status_payload" 2>"$status_err")"; then
@@ -254,7 +285,7 @@ jobs:
            --arg source_event "issue_comment" \
            --arg source_action "$SOURCE_ACTION" \
            '{event_type:"clawsweeper_comment",client_payload:({target_repo:$target_repo,item_number:$item_number,comment_id:$comment_id,source_event:$source_event,source_action:$source_action,max_comments:"1"} + (if $status_comment_id != "" then {status_comment_id:($status_comment_id|tonumber)} else {} end))}')"
-          if GH_TOKEN="$DISPATCH_TOKEN" gh api repos/openclaw/clawsweeper/dispatches \
+          if GH_TOKEN="$DISPATCH_TOKEN" gh_api_with_retry repos/openclaw/clawsweeper/dispatches \
            --method POST \
            --input - <<< "$payload"; then
            echo "Dispatched ClawSweeper comment router."
@@ -276,6 +307,7 @@ jobs:
            echo "::notice::Skipping ClawSweeper commit dispatch because no ClawSweeper app token is configured. Not falling back to a maintainer token."
            exit 0
          fi
+          . "$RUNNER_TEMP/github-api-backoff.sh"
          case "$CREATE_CHECKS" in
            true|TRUE|1|yes|YES|on|ON) create_checks=true ;;
            *) create_checks=false ;;
@@ -287,7 +319,7 @@ jobs:
            --arg ref "$SOURCE_REF" \
            --argjson create_checks "$create_checks" \
            '{event_type:"clawsweeper_commit_review",client_payload:{target_repo:$target_repo,before_sha:$before_sha,after_sha:$after_sha,ref:$ref,enabled:true,create_checks:$create_checks}}')"
-          if gh api repos/openclaw/clawsweeper/dispatches \
+          if gh_api_with_retry repos/openclaw/clawsweeper/dispatches \
            --method POST \
            --input - <<< "$payload"; then
            echo "Dispatched ClawSweeper commit review."
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -33,7 +33,7 @@ on:

 concurrency:
  group: codeql-${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('ref-{0}', github.ref) }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}

 env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
--- a/.github/workflows/control-ui-locale-refresh.yml
+++ b/.github/workflows/control-ui-locale-refresh.yml
@@ -23,8 +23,8 @@ permissions:
  contents: write

 concurrency:
-  group: control-ui-locale-refresh
-  cancel-in-progress: false
+  group: control-ui-locale-refresh-${{ github.event_name == 'push' && github.ref || github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.event_name == 'release' && format('release-{0}', github.event.release.tag_name) || format('{0}-{1}', github.event_name, github.run_id) }}
+  cancel-in-progress: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}

 jobs:
  plan:
--- a/.github/workflows/docs-sync-publish.yml
+++ b/.github/workflows/docs-sync-publish.yml
@@ -13,6 +13,10 @@ on:
 permissions:
  contents: read

+concurrency:
+  group: docs-sync-publish-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+
 jobs:
  sync-publish-repo:
    runs-on: ubuntu-latest
--- a/.github/workflows/openclaw-stable-main-closeout.yml
+++ b/.github/workflows/openclaw-stable-main-closeout.yml
@@ -23,8 +23,8 @@ permissions:
  contents: write

 concurrency:
-  group: openclaw-stable-main-closeout
-  cancel-in-progress: false
+  group: openclaw-stable-main-closeout-${{ github.event_name == 'workflow_dispatch' && (inputs.tag || github.run_id) || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}

 jobs:
  resolve:
@@ -43,6 +43,30 @@ jobs:
      should_closeout: ${{ steps.inputs.outputs.should_closeout }}
      tag: ${{ steps.inputs.outputs.tag }}
    steps:
+      - name: Install GitHub API backoff helper
+        run: |
+          cat > "$RUNNER_TEMP/github-api-backoff.sh" <<'BASH'
+          gh_with_retry() {
+            local attempt output status lower_output
+            for attempt in 1 2 3 4 5; do
+              if output="$(gh "$@" 2>&1)"; then
+                printf '%s\n' "$output"
+                return 0
+              fi
+              status=$?
+              lower_output="${output,,}"
+              if [[ "$lower_output" != *"rate limit"* && "$output" != *"HTTP 429"* ]]; then
+                printf '%s\n' "$output" >&2
+                return "$status"
+              fi
+              echo "::warning::GitHub API throttled stable closeout on attempt ${attempt}; retrying after backoff." >&2
+              sleep $((attempt * attempt * 5))
+            done
+            printf '%s\n' "$output" >&2
+            return "$status"
+          }
+          BASH
+
      - name: Checkout pushed main
        if: ${{ github.event_name == 'push' }}
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -62,9 +86,13 @@ jobs:
          TRIGGER_SHA: ${{ github.sha }}
        run: |
          set -euo pipefail
+          if [[ "$EVENT_NAME" == "push" ]]; then
+            sleep 45
+          fi
+          . "$RUNNER_TEMP/github-api-backoff.sh"
          if [[ "$EVENT_NAME" == "push" ]]; then
            main_ref="$TRIGGER_SHA"
-            tag="$(gh release list --repo "$GITHUB_REPOSITORY" --exclude-drafts --limit 100 \
+            tag="$(gh_with_retry release list --repo "$GITHUB_REPOSITORY" --exclude-drafts --limit 100 \
              --json tagName,isPrerelease,publishedAt \
              --jq '[.[] | select(.isPrerelease | not) | select(.tagName | test("^v[0-9]{4}\\.[0-9]+\\.[0-9]+(-[0-9]+)?$"))] | sort_by(.publishedAt) | last | .tagName // empty')"
            if [[ -z "$tag" ]]; then
@@ -91,7 +119,7 @@ jobs:
          tag_package_content="$RUNNER_TEMP/tag-package-content.b64"
          tag_package_read=false
          for attempt in 1 2 3; do
-            if gh api "repos/$GITHUB_REPOSITORY/contents/package.json?ref=$tag" \
+            if gh_with_retry api "repos/$GITHUB_REPOSITORY/contents/package.json?ref=$tag" \
              --jq '.content' > "$tag_package_content"; then
              tag_package_read=true
              break
@@ -126,7 +154,7 @@ jobs:
          closeout_checksum_asset="${closeout_asset}.sha256"
          closeout_dir="$RUNNER_TEMP/release-closeout-evidence"
          mkdir -p "$closeout_dir"
-          gh release download "$tag" --repo "$GITHUB_REPOSITORY" \
+          gh_with_retry release download "$tag" --repo "$GITHUB_REPOSITORY" \
            --pattern "$closeout_asset" --pattern "$closeout_checksum_asset" --dir "$closeout_dir" || true
          closeout_json_path="$closeout_dir/$closeout_asset"
          closeout_checksum_path="$closeout_dir/$closeout_checksum_asset"
@@ -182,7 +210,7 @@ jobs:
          fi
          evidence_dir="$RUNNER_TEMP/release-postpublish-evidence"
          mkdir -p "$evidence_dir"
-          if ! gh release download "$evidence_source_tag" --repo "$GITHUB_REPOSITORY" \
+          if ! gh_with_retry release download "$evidence_source_tag" --repo "$GITHUB_REPOSITORY" \
            --pattern "$evidence_asset" --pattern "$evidence_checksum_asset" --dir "$evidence_dir"; then
            if [[ "$EVENT_NAME" == "push" ]]; then
              echo "Stable closeout skipped: $evidence_source_tag predates immutable postpublish evidence." >&2
@@ -272,6 +300,30 @@ jobs:
            exit 1
          fi

+      - name: Install GitHub API backoff helper
+        run: |
+          cat > "$RUNNER_TEMP/github-api-backoff.sh" <<'BASH'
+          gh_with_retry() {
+            local attempt output status lower_output
+            for attempt in 1 2 3 4 5; do
+              if output="$(gh "$@" 2>&1)"; then
+                printf '%s\n' "$output"
+                return 0
+              fi
+              status=$?
+              lower_output="${output,,}"
+              if [[ "$lower_output" != *"rate limit"* && "$output" != *"HTTP 429"* ]]; then
+                printf '%s\n' "$output" >&2
+                return "$status"
+              fi
+              echo "::warning::GitHub API throttled stable closeout on attempt ${attempt}; retrying after backoff." >&2
+              sleep $((attempt * attempt * 5))
+            done
+            printf '%s\n' "$output" >&2
+            return "$status"
+          }
+          BASH
+
      - name: Verify release workflow evidence
        env:
          GH_TOKEN: ${{ github.token }}
@@ -279,7 +331,8 @@ jobs:
          RELEASE_PUBLISH_RUN_ID: ${{ needs.resolve.outputs.release_publish_run_id }}
        run: |
          set -euo pipefail
-          gh run view "$FULL_RELEASE_VALIDATION_RUN_ID" --repo "$GITHUB_REPOSITORY" \
+          . "$RUNNER_TEMP/github-api-backoff.sh"
+          gh_with_retry run view "$FULL_RELEASE_VALIDATION_RUN_ID" --repo "$GITHUB_REPOSITORY" \
            --json workflowName,event,status,conclusion \
            > "$RUNNER_TEMP/full-release-validation-run.json"
          node --input-type=module - "$RUNNER_TEMP/full-release-validation-run.json" <<'NODE'
@@ -296,7 +349,7 @@ jobs:
            }
          }
          NODE
-          gh run view "$RELEASE_PUBLISH_RUN_ID" --repo "$GITHUB_REPOSITORY" \
+          gh_with_retry run view "$RELEASE_PUBLISH_RUN_ID" --repo "$GITHUB_REPOSITORY" \
            --json workflowName,event,status,conclusion \
            > "$RUNNER_TEMP/release-publish-run.json"
          node --input-type=module - "$RUNNER_TEMP/release-publish-run.json" <<'NODE'
@@ -317,7 +370,7 @@ jobs:
          manifest_dir="$RUNNER_TEMP/full-release-validation-manifest"
          rm -rf "$manifest_dir"
          mkdir -p "$manifest_dir"
-          gh run download "$FULL_RELEASE_VALIDATION_RUN_ID" --repo "$GITHUB_REPOSITORY" \
+          gh_with_retry run download "$FULL_RELEASE_VALIDATION_RUN_ID" --repo "$GITHUB_REPOSITORY" \
            --name "full-release-validation-${FULL_RELEASE_VALIDATION_RUN_ID}" \
            --dir "$manifest_dir"
          tag_sha="$(git -C "$GITHUB_WORKSPACE/release-tag" rev-parse HEAD)"
@@ -346,7 +399,8 @@ jobs:
        run: |
          set -euo pipefail
          mkdir -p "$CLOSEOUT_DIR"
-          gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
+          . "$RUNNER_TEMP/github-api-backoff.sh"
+          gh_with_retry release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
            --json tagName,isDraft,isPrerelease,assets \
            > "$CLOSEOUT_DIR/github-release.json"
          node scripts/verify-stable-main-closeout.mjs \
@@ -372,13 +426,14 @@ jobs:
          CLOSEOUT_DIR: ${{ runner.temp }}/openclaw-stable-main-closeout
        run: |
          set -euo pipefail
+          . "$RUNNER_TEMP/github-api-backoff.sh"
          release_version="${RELEASE_TAG#v}"
          attach_or_verify() {
            local source_path="$1"
            local asset_name="$2"
            local existing_dir="$CLOSEOUT_DIR/existing-${asset_name}"
            mkdir -p "$existing_dir"
-            if gh release download "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
+            if gh_with_retry release download "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \
              --pattern "$asset_name" --dir "$existing_dir"; then
              cmp --silent "$source_path" "$existing_dir/$asset_name" || {
                echo "Existing release asset $asset_name differs from closeout evidence." >&2
@@ -386,7 +441,7 @@ jobs:
              }
              return
            fi
-            gh release upload "$RELEASE_TAG" "$source_path#$asset_name" --repo "$GITHUB_REPOSITORY"
+            gh_with_retry release upload "$RELEASE_TAG" "$source_path#$asset_name" --repo "$GITHUB_REPOSITORY"
          }
          attach_or_verify \
            "$CLOSEOUT_DIR/stable-main-closeout.json" \
--- a/.github/workflows/sandbox-common-smoke.yml
+++ b/.github/workflows/sandbox-common-smoke.yml
@@ -19,7 +19,7 @@ permissions:

 concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main') }}

 env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"