Fix Playwright report comments on forked PRs (#1975)

## Summary
- update the Playwright summary script to support workflow_run events
and optional comment skipping
- stop the CI workflow from posting Playwright comments directly and
only generate the summary
- add a workflow_run-based commenter workflow that downloads artifacts
and posts results for PRs, including forks

## Testing
- not run (workflow changes only)


------
[Codex
Task](https://chatgpt.com/codex/tasks/task_e_694340b2da6083278e42db076ea89eba)

<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> Moves Playwright commenting to a workflow_run job that downloads
artifacts and posts/updates the summary; CI now only uploads the report,
and the summary script improves PR/run detection and OS bucketing.
> 
> - **Workflows**:
> - **CI (`.github/workflows/ci.yml`)**: Remove in-job PR comment step;
keep merging reports and uploading `playwright-report` artifact.
> - **New (`.github/workflows/playwright-comment.yml`)**: `workflow_run`
on CI to download artifacts (`html-report--attempt-*`, `blob-report-*`)
and run `scripts/generate-playwright-summary.js` to comment on the PR
(supports forks).
> - **Script (`scripts/generate-playwright-summary.js`)**:
> - Add PR detection for `workflow_run` and `PR_NUMBER`; use
`PLAYWRIGHT_RUN_ID` for report link.
> - Improve OS detection/bucketing (auto-detect from attachments/stacks,
sensible defaults, lazy bucket creation).
> - Safer fallbacks when no artifacts; always write job summary; skip PR
comment when no PR is detected.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
8428f7ad6eb0671571cb4ae0e473434ffb1cf8d1. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->



<!-- This is an auto-generated description by cubic. -->
---
## Summary by cubic
Fixes Playwright report comments on forked PRs by moving comment posting
to a workflow_run job. CI now only uploads the reports; a separate
workflow posts or updates the PR comment using artifacts.

- **Bug Fixes**
- Added Playwright Report Comment workflow (workflow_run on CI) to
download artifacts and comment on PRs from forks.
  - Removed PR comment step from CI; CI only uploads Playwright reports.
- Updated summary script to support workflow_run, auto-detect the PR
number, use PLAYWRIGHT_RUN_ID for links, and improve OS
detection/bucketing.

<sup>Written for commit 8428f7ad6eb0671571cb4ae0e473434ffb1cf8d1.
Summary will update automatically on new commits.</sup>

<!-- End of auto-generated description by cubic. -->

.github/workflows/ci.yml

@@ -151,10 +151,3 @@ jobs:
           name: html-report--attempt-${{ github.run_attempt }}
           path: playwright-report
           retention-days: 3
-
-      - name: Generate test summary comment
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const { run } = require('./scripts/generate-playwright-summary.js');
-            await run({ github, context, core });

.github/workflows/playwright-comment.yml

@@ -0,0 +1,59 @@
+name: Playwright Report Comment
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+  pull-requests: write
+  issues: write
+  actions: read
+
+jobs:
+  comment:
+    if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.pull_requests[0].number }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.workflow_run.base_ref }}
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: Download Playwright HTML report
+        uses: actions/download-artifact@v4
+        with:
+          name: html-report--attempt-${{ github.event.workflow_run.run_attempt }}
+          path: playwright-report
+          github-token: ${{ github.token }}
+          repository: ${{ github.event.workflow_run.repository.full_name }}
+          run-id: ${{ github.event.workflow_run.id }}
+          if-no-artifact-found: warn
+
+      - name: Download blob reports
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+          github-token: ${{ github.token }}
+          repository: ${{ github.event.workflow_run.repository.full_name }}
+          run-id: ${{ github.event.workflow_run.id }}
+          if-no-artifact-found: warn
+
+      - name: Generate Playwright summary comment
+        uses: actions/github-script@v7
+        env:
+          PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }}
+          PLAYWRIGHT_RUN_ID: ${{ github.event.workflow_run.id }}
+        with:
+          script: |
+            const { run } = require('./scripts/generate-playwright-summary.js');
+            await run({ github, context, core });

scripts/generate-playwright-summary.js

@@ -10,6 +10,73 @@ function stripAnsi(str) {
   return str.replace(/\x1b\[[0-9;]*m/g, "").replace(/\u001b\[[0-9;]*m/g, "");
 }
 
+function ensureOsBucket(resultsByOs, os) {
+  if (!os) return;
+  if (!resultsByOs[os]) {
+    resultsByOs[os] = {
+      passed: 0,
+      failed: 0,
+      skipped: 0,
+      flaky: 0,
+      failures: [],
+      flakyTests: [],
+    };
+  }
+}
+
+function detectOperatingSystemsFromReport(report) {
+  const detected = new Set();
+
+  function traverseSuites(suites = []) {
+    for (const suite of suites) {
+      for (const spec of suite.specs || []) {
+        for (const test of spec.tests || []) {
+          for (const result of test.results || []) {
+            for (const attachment of result.attachments || []) {
+              const p = attachment.path || "";
+              if (p.includes("darwin") || p.includes("macos")) {
+                detected.add("macOS");
+              } else if (p.includes("win32") || p.includes("windows")) {
+                detected.add("Windows");
+              }
+            }
+
+            const stack = result.error?.stack || "";
+            if (stack.includes("/Users/")) {
+              detected.add("macOS");
+            } else if (stack.includes("C:\\") || stack.includes("D:\\")) {
+              detected.add("Windows");
+            }
+          }
+        }
+      }
+
+      if (suite.suites?.length) {
+        traverseSuites(suite.suites);
+      }
+    }
+  }
+
+  traverseSuites(report?.suites);
+
+  return detected;
+}
+
+function determineIssueNumber({ context }) {
+  const envNumber = process.env.PR_NUMBER;
+  if (envNumber) return Number(envNumber);
+
+  if (context.eventName === "workflow_run") {
+    const prFromPayload =
+      context.payload?.workflow_run?.pull_requests?.[0]?.number;
+    if (prFromPayload) return prFromPayload;
+  } else {
+    throw new Error("This script should only be run in a workflow_run")
+  }
+
+  return null;
+}
+
 async function run({ github, context, core }) {
   // Read the JSON report
   const reportPath = "playwright-report/results.json";
@@ -28,24 +95,18 @@ async function run({ github, context, core }) {
 
   // Initialize per-OS results
   const resultsByOs = {};
-  if (hasMacOS)
+  if (hasMacOS) ensureOsBucket(resultsByOs, "macOS");
-    resultsByOs["macOS"] = {
+  if (hasWindows) ensureOsBucket(resultsByOs, "Windows");
-      passed: 0,
+
-      failed: 0,
+  if (Object.keys(resultsByOs).length === 0) {
-      skipped: 0,
+    const detected = detectOperatingSystemsFromReport(report);
-      flaky: 0,
+    if (detected.size === 0) {
-      failures: [],
+      ensureOsBucket(resultsByOs, "macOS");
-      flakyTests: [],
+      ensureOsBucket(resultsByOs, "Windows");
-    };
+    } else {
-  if (hasWindows)
+      for (const os of detected) ensureOsBucket(resultsByOs, os);
-    resultsByOs["Windows"] = {
+    }
-      passed: 0,
+  }
-      failed: 0,
-      skipped: 0,
-      flaky: 0,
-      failures: [],
-      flakyTests: [],
-    };
 
   // Traverse suites and collect test results
   function traverseSuites(suites, parentTitle = "") {
@@ -94,7 +155,11 @@ async function run({ github, context, core }) {
           }
 
           // If we still don't know, assign to both (will be roughly split)
-          const osTargets = os ? [os] : Object.keys(resultsByOs);
+          const osTargets = os
+            ? [os]
+            : Object.keys(resultsByOs).length > 0
+              ? Object.keys(resultsByOs)
+              : ["macOS", "Windows"];
 
           // Check if this is a flaky test (passed eventually but had prior failures)
           const hadPriorFailure = results
@@ -108,7 +173,7 @@ async function run({ github, context, core }) {
           const isFlaky = finalResult.status === "passed" && hadPriorFailure;
 
           for (const targetOs of osTargets) {
-            if (!resultsByOs[targetOs]) continue;
+            ensureOsBucket(resultsByOs, targetOs);
             const status = finalResult.status;
 
             if (isFlaky) {
@@ -240,15 +305,17 @@ async function run({ github, context, core }) {
   }
 
   const repoUrl = `https://github.com/${process.env.GITHUB_REPOSITORY}`;
-  const runId = process.env.GITHUB_RUN_ID;
+  const runId = process.env.PLAYWRIGHT_RUN_ID || process.env.GITHUB_RUN_ID;
   comment += `\n---\n📊 [View full report](${repoUrl}/actions/runs/${runId})`;
 
   // Post or update comment on PR
-  if (context.eventName === "pull_request") {
+  const prNumber = determineIssueNumber({ context });
+
+  if (prNumber) {
     const { data: comments } = await github.rest.issues.listComments({
       owner: context.repo.owner,
       repo: context.repo.repo,
-      issue_number: context.issue.number,
+      issue_number: prNumber,
     });
 
     const botComment = comments.find(
@@ -268,10 +335,12 @@ async function run({ github, context, core }) {
       await github.rest.issues.createComment({
         owner: context.repo.owner,
         repo: context.repo.repo,
-        issue_number: context.issue.number,
+        issue_number: prNumber,
         body: comment,
       });
     }
+  } else if (!prNumber) {
+    console.log("No pull request detected; skipping PR comment");
   }
 
   // Always output to job summary