From 7cf8317f55d00bf55dbcd1c7073459b5f7a3dded Mon Sep 17 00:00:00 2001 From: Will Chen Date: Wed, 17 Dec 2025 17:30:58 -0800 Subject: [PATCH] Fix Playwright report comments on forked PRs (#1975) ## Summary - update the Playwright summary script to support workflow_run events and optional comment skipping - stop the CI workflow from posting Playwright comments directly and only generate the summary - add a workflow_run-based commenter workflow that downloads artifacts and posts results for PRs, including forks ## Testing - not run (workflow changes only) ------ [Codex Task](https://chatgpt.com/codex/tasks/task_e_694340b2da6083278e42db076ea89eba) --- > [!NOTE] > Moves Playwright commenting to a workflow_run job that downloads artifacts and posts/updates the summary; CI now only uploads the report, and the summary script improves PR/run detection and OS bucketing. > > - **Workflows**: > - **CI (`.github/workflows/ci.yml`)**: Remove in-job PR comment step; keep merging reports and uploading `playwright-report` artifact. > - **New (`.github/workflows/playwright-comment.yml`)**: `workflow_run` on CI to download artifacts (`html-report--attempt-*`, `blob-report-*`) and run `scripts/generate-playwright-summary.js` to comment on the PR (supports forks). > - **Script (`scripts/generate-playwright-summary.js`)**: > - Add PR detection for `workflow_run` and `PR_NUMBER`; use `PLAYWRIGHT_RUN_ID` for report link. > - Improve OS detection/bucketing (auto-detect from attachments/stacks, sensible defaults, lazy bucket creation). > - Safer fallbacks when no artifacts; always write job summary; skip PR comment when no PR is detected. > > Written by [Cursor Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit 8428f7ad6eb0671571cb4ae0e473434ffb1cf8d1. This will update automatically on new commits. Configure [here](https://cursor.com/dashboard?tab=bugbot). --- ## Summary by cubic Fixes Playwright report comments on forked PRs by moving comment posting to a workflow_run job. CI now only uploads the reports; a separate workflow posts or updates the PR comment using artifacts. - **Bug Fixes** - Added Playwright Report Comment workflow (workflow_run on CI) to download artifacts and comment on PRs from forks. - Removed PR comment step from CI; CI only uploads Playwright reports. - Updated summary script to support workflow_run, auto-detect the PR number, use PLAYWRIGHT_RUN_ID for links, and improve OS detection/bucketing. Written for commit 8428f7ad6eb0671571cb4ae0e473434ffb1cf8d1. Summary will update automatically on new commits. --- .github/workflows/ci.yml | 7 -- .github/workflows/playwright-comment.yml | 59 ++++++++++++ scripts/generate-playwright-summary.js | 117 ++++++++++++++++++----- 3 files changed, 152 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/playwright-comment.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f533288..7f7c8c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,10 +151,3 @@ jobs: name: html-report--attempt-${{ github.run_attempt }} path: playwright-report retention-days: 3 - - - name: Generate test summary comment - uses: actions/github-script@v7 - with: - script: | - const { run } = require('./scripts/generate-playwright-summary.js'); - await run({ github, context, core }); diff --git a/.github/workflows/playwright-comment.yml b/.github/workflows/playwright-comment.yml new file mode 100644 index 0000000..1a23c0d --- /dev/null +++ b/.github/workflows/playwright-comment.yml @@ -0,0 +1,59 @@ +name: Playwright Report Comment + +on: + workflow_run: + workflows: ["CI"] + types: + - completed + +permissions: + contents: read + pull-requests: write + issues: write + actions: read + +jobs: + comment: + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].number }} + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.base_ref }} + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: lts/* + + - name: Download Playwright HTML report + uses: actions/download-artifact@v4 + with: + name: html-report--attempt-${{ github.event.workflow_run.run_attempt }} + path: playwright-report + github-token: ${{ github.token }} + repository: ${{ github.event.workflow_run.repository.full_name }} + run-id: ${{ github.event.workflow_run.id }} + if-no-artifact-found: warn + + - name: Download blob reports + uses: actions/download-artifact@v4 + with: + path: all-blob-reports + pattern: blob-report-* + merge-multiple: true + github-token: ${{ github.token }} + repository: ${{ github.event.workflow_run.repository.full_name }} + run-id: ${{ github.event.workflow_run.id }} + if-no-artifact-found: warn + + - name: Generate Playwright summary comment + uses: actions/github-script@v7 + env: + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + PLAYWRIGHT_RUN_ID: ${{ github.event.workflow_run.id }} + with: + script: | + const { run } = require('./scripts/generate-playwright-summary.js'); + await run({ github, context, core }); diff --git a/scripts/generate-playwright-summary.js b/scripts/generate-playwright-summary.js index 4f17284..5af16c3 100644 --- a/scripts/generate-playwright-summary.js +++ b/scripts/generate-playwright-summary.js @@ -10,6 +10,73 @@ function stripAnsi(str) { return str.replace(/\x1b\[[0-9;]*m/g, "").replace(/\u001b\[[0-9;]*m/g, ""); } +function ensureOsBucket(resultsByOs, os) { + if (!os) return; + if (!resultsByOs[os]) { + resultsByOs[os] = { + passed: 0, + failed: 0, + skipped: 0, + flaky: 0, + failures: [], + flakyTests: [], + }; + } +} + +function detectOperatingSystemsFromReport(report) { + const detected = new Set(); + + function traverseSuites(suites = []) { + for (const suite of suites) { + for (const spec of suite.specs || []) { + for (const test of spec.tests || []) { + for (const result of test.results || []) { + for (const attachment of result.attachments || []) { + const p = attachment.path || ""; + if (p.includes("darwin") || p.includes("macos")) { + detected.add("macOS"); + } else if (p.includes("win32") || p.includes("windows")) { + detected.add("Windows"); + } + } + + const stack = result.error?.stack || ""; + if (stack.includes("/Users/")) { + detected.add("macOS"); + } else if (stack.includes("C:\\") || stack.includes("D:\\")) { + detected.add("Windows"); + } + } + } + } + + if (suite.suites?.length) { + traverseSuites(suite.suites); + } + } + } + + traverseSuites(report?.suites); + + return detected; +} + +function determineIssueNumber({ context }) { + const envNumber = process.env.PR_NUMBER; + if (envNumber) return Number(envNumber); + + if (context.eventName === "workflow_run") { + const prFromPayload = + context.payload?.workflow_run?.pull_requests?.[0]?.number; + if (prFromPayload) return prFromPayload; + } else { + throw new Error("This script should only be run in a workflow_run") + } + + return null; +} + async function run({ github, context, core }) { // Read the JSON report const reportPath = "playwright-report/results.json"; @@ -28,24 +95,18 @@ async function run({ github, context, core }) { // Initialize per-OS results const resultsByOs = {}; - if (hasMacOS) - resultsByOs["macOS"] = { - passed: 0, - failed: 0, - skipped: 0, - flaky: 0, - failures: [], - flakyTests: [], - }; - if (hasWindows) - resultsByOs["Windows"] = { - passed: 0, - failed: 0, - skipped: 0, - flaky: 0, - failures: [], - flakyTests: [], - }; + if (hasMacOS) ensureOsBucket(resultsByOs, "macOS"); + if (hasWindows) ensureOsBucket(resultsByOs, "Windows"); + + if (Object.keys(resultsByOs).length === 0) { + const detected = detectOperatingSystemsFromReport(report); + if (detected.size === 0) { + ensureOsBucket(resultsByOs, "macOS"); + ensureOsBucket(resultsByOs, "Windows"); + } else { + for (const os of detected) ensureOsBucket(resultsByOs, os); + } + } // Traverse suites and collect test results function traverseSuites(suites, parentTitle = "") { @@ -94,7 +155,11 @@ async function run({ github, context, core }) { } // If we still don't know, assign to both (will be roughly split) - const osTargets = os ? [os] : Object.keys(resultsByOs); + const osTargets = os + ? [os] + : Object.keys(resultsByOs).length > 0 + ? Object.keys(resultsByOs) + : ["macOS", "Windows"]; // Check if this is a flaky test (passed eventually but had prior failures) const hadPriorFailure = results @@ -108,7 +173,7 @@ async function run({ github, context, core }) { const isFlaky = finalResult.status === "passed" && hadPriorFailure; for (const targetOs of osTargets) { - if (!resultsByOs[targetOs]) continue; + ensureOsBucket(resultsByOs, targetOs); const status = finalResult.status; if (isFlaky) { @@ -240,15 +305,17 @@ async function run({ github, context, core }) { } const repoUrl = `https://github.com/${process.env.GITHUB_REPOSITORY}`; - const runId = process.env.GITHUB_RUN_ID; + const runId = process.env.PLAYWRIGHT_RUN_ID || process.env.GITHUB_RUN_ID; comment += `\n---\nšŸ“Š [View full report](${repoUrl}/actions/runs/${runId})`; // Post or update comment on PR - if (context.eventName === "pull_request") { + const prNumber = determineIssueNumber({ context }); + + if (prNumber) { const { data: comments } = await github.rest.issues.listComments({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: context.issue.number, + issue_number: prNumber, }); const botComment = comments.find( @@ -268,10 +335,12 @@ async function run({ github, context, core }) { await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, - issue_number: context.issue.number, + issue_number: prNumber, body: comment, }); } + } else if (!prNumber) { + console.log("No pull request detected; skipping PR comment"); } // Always output to job summary