From 5a7295ff5d5cfc222d89aa61e14b716a05abd9a7 Mon Sep 17 00:00:00 2001 From: adilallo <39313955+adilallo@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:01:52 -0600 Subject: [PATCH] Implement comprehensive visual regression stability improvements --- .gitea/workflows/ci.yaml | 82 +++++++++++++++++++---------- playwright.config.ts | 17 ++++-- tests/e2e/visual-regression.spec.ts | 55 ++++++++++++++++++- 3 files changed, 120 insertions(+), 34 deletions(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index da06f17..d2bf66e 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -47,8 +47,15 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: { node-version: 20, cache: npm } + - name: Cache Playwright + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - run: npm ci - - run: npx playwright install --with-deps ${{ matrix.browser }} + - run: npx playwright install ${{ matrix.browser }} + env: + PLAYWRIGHT_BROWSERS_PATH: ~/.cache/ms-playwright - run: npm run build - name: E2E (start + test + teardown) @@ -80,13 +87,6 @@ jobs: echo "๐Ÿงช Running E2E tests for ${{ matrix.browser }}..." BASE_URL="http://$HOST:$PORT" npx playwright test --project=${{ matrix.browser }} --reporter=list || TEST_EXIT_CODE=$? - # Generate baseline snapshots only on main branch in CI environment - if [ "${{ gitea.ref }}" = "refs/heads/main" ]; then - echo "๐ŸŒฑ Generating baseline snapshots for ${{ matrix.browser }} in CI environment..." - PLAYWRIGHT_UPDATE_SNAPSHOTS=1 BASE_URL="http://$HOST:$PORT" npx playwright test tests/e2e/visual-regression.spec.ts --project=${{ matrix.browser }} - echo "โœ… Baseline snapshots generated for ${{ matrix.browser }} in CI" - fi - # Teardown echo "๐Ÿงน Cleaning up server..." kill "$SVPID" 2>/dev/null || true @@ -110,28 +110,21 @@ jobs: path: playwright-${{ matrix.browser }}.tgz retention-days: 30 - - name: Commit baseline snapshots (if generated) - if: gitea.ref == 'refs/heads/main' && always() - run: | - if [ -n "$(git status --porcelain tests/e2e/visual-regression.spec.ts-snapshots/)" ]; then - echo "๐Ÿ”„ Committing baseline snapshots for ${{ matrix.browser }} generated in CI..." - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add tests/e2e/visual-regression.spec.ts-snapshots/ - git commit -m "Generate baseline snapshots for ${{ matrix.browser }} in CI environment" || true - echo "โœ… Baseline snapshots committed for ${{ matrix.browser }}" - else - echo "โ„น๏ธ No new baseline snapshots to commit for ${{ matrix.browser }}" - fi - visual-regression: runs-on: [self-hosted, macos-latest] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: { node-version: 20, cache: npm } + - name: Cache Playwright + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - run: npm ci - - run: npx playwright install --with-deps + - run: npx playwright install + env: + PLAYWRIGHT_BROWSERS_PATH: ~/.cache/ms-playwright - run: npm run build # 1) Sanity check that the build exists - name: Verify Next build output @@ -165,11 +158,7 @@ jobs: curl -fsS "http://$HOST:$PORT" >/dev/null echo "โœ… App is responding at http://$HOST:$PORT" - # Seed snapshots on main branch only (one-time setup) - if [ "${{ gitea.ref }}" = "refs/heads/main" ]; then - echo "๐ŸŒฑ Seeding snapshots on main branch..." - PLAYWRIGHT_UPDATE_SNAPSHOTS=1 npx playwright test tests/e2e/visual-regression.spec.ts --project=chromium - fi + # Run visual regression tests echo "๐Ÿงช Running visual regression tests..." @@ -293,6 +282,43 @@ jobs: name: lhci-results path: lhci-results + seed-vr-snapshots: + if: gitea.ref == 'refs/heads/main' + runs-on: [self-hosted, macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: 20, cache: npm } + - name: Cache Playwright + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ms-playwright-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + - run: npm ci + - run: npx playwright install + env: + PLAYWRIGHT_BROWSERS_PATH: ~/.cache/ms-playwright + - run: npm run build + - name: Start app + wait + run: | + node node_modules/next/dist/bin/next start -p 3010 -H 127.0.0.1 > .next/runner.log 2>&1 & + npx wait-on -t 120000 tcp:127.0.0.1:3010 + - name: Generate snapshots for ALL projects + env: + { + PLAYWRIGHT_UPDATE_SNAPSHOTS: "1", + BASE_URL: "http://127.0.0.1:3010", + } + run: npx playwright test tests/e2e/visual-regression.spec.ts --project=chromium --project=firefox --project=webkit --project=mobile + - name: Commit snapshots + run: | + if [ -n "$(git status --porcelain tests/e2e/visual-regression.spec.ts-snapshots/)" ]; then + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add tests/e2e/visual-regression.spec.ts-snapshots/ + git commit -m "Seed Playwright VR snapshots (CI, all projects)" + fi + storybook: runs-on: [self-hosted, macos-latest] steps: diff --git a/playwright.config.ts b/playwright.config.ts index 9c628c9..d2b587e 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -22,7 +22,6 @@ export default defineConfig({ // Deterministic rendering defaults to eliminate environment drift colorScheme: "light", viewport: { width: 1280, height: 800 }, - deviceScaleFactor: 1, // Keep DPR=1 for stable anti-aliasing timezoneId: "UTC", // Freeze timezone locale: "en-US", // Freeze locale headless: true, @@ -46,28 +45,36 @@ export default defineConfig({ name: "chromium", use: { ...devices["Desktop Chrome"], - deviceScaleFactor: 1, // Override device scale for consistency + // Let device preset own the DPR for stable anti-aliasing + launchOptions: { + args: [ + "--force-color-profile=srgb", + "--disable-skia-runtime-opts", + "--font-render-hinting=none", + "--disable-lcd-text", + ], + }, }, }, { name: "firefox", use: { ...devices["Desktop Firefox"], - deviceScaleFactor: 1, // Override device scale for consistency + // Let device preset own the DPR for stable anti-aliasing }, }, { name: "webkit", use: { ...devices["Desktop Safari"], - deviceScaleFactor: 1, // Override device scale for consistency + // Let device preset own the DPR for stable anti-aliasing }, }, { name: "mobile", use: { ...devices["iPhone 13"], - deviceScaleFactor: 1, // Override device scale for consistency + // Let device preset own the DPR for stable anti-aliasing }, }, ], diff --git a/tests/e2e/visual-regression.spec.ts b/tests/e2e/visual-regression.spec.ts index 96d45bf..e41d19c 100644 --- a/tests/e2e/visual-regression.spec.ts +++ b/tests/e2e/visual-regression.spec.ts @@ -1,21 +1,67 @@ import { test, expect } from "@playwright/test"; test.describe("Visual Regression Tests", () => { + async function settle(page: any) { + await page.evaluate(() => { + window.scrollTo(0, window.scrollY); // ensure a frame boundary + void document.body.getBoundingClientRect(); + }); + await page.waitForTimeout(50); + } test.beforeEach(async ({ page }) => { + // Add deterministic CSS to normalize rendering + await page.addStyleTag({ + content: ` + /* stop caret and selection flicker */ + * { caret-color: transparent !important; } + ::selection { background: transparent !important; } + /* hide scrollbars */ + ::-webkit-scrollbar { display: none !important; } + html { scrollbar-width: none !important; } + /* stabilize font rasterization */ + * { + text-rendering: geometricPrecision !important; + -webkit-font-smoothing: antialiased !important; + -moz-osx-font-smoothing: grayscale !important; + } + `, + }); + await page.goto("/"); // Wait for all content to load await page.waitForLoadState("networkidle"); + + // Make sure we've really got the webfonts before shots + await page.evaluate(async () => { + // @ts-ignore + if (document.fonts && document.fonts.status !== "loaded") { + // @ts-ignore + await document.fonts.ready; + } + }); }); test("homepage full page screenshot", async ({ page }) => { + // Stabilize layout before screenshot + await settle(page); + // Take full page screenshot await expect(page).toHaveScreenshot("homepage-full.png", { fullPage: true, animations: "disabled", + scale: "css", }); }); test("homepage viewport screenshot", async ({ page }) => { + // Stabilize layout before screenshot + await page.evaluate(() => { + window.scrollTo(0, 0); + // Force layout & a frame boundary + void document.body.getBoundingClientRect(); + }); + await page.waitForTimeout(50); // give the compositor one tick + // Take viewport screenshot await expect(page).toHaveScreenshot("homepage-viewport.png", { animations: "disabled", @@ -27,6 +73,13 @@ test.describe("Visual Regression Tests", () => { await page.locator("text=Collaborate").scrollIntoViewIfNeeded(); await page.waitForTimeout(500); // Wait for animations + // Stabilize layout before screenshot + await page.evaluate(() => { + // Force layout & a frame boundary + void document.body.getBoundingClientRect(); + }); + await page.waitForTimeout(50); // give the compositor one tick + const heroSection = page.locator("section").first(); await expect(heroSection).toHaveScreenshot("hero-banner.png", { animations: "disabled", @@ -360,7 +413,7 @@ test.describe("Visual Regression Tests", () => { await page.evaluate(() => { document.documentElement.style.setProperty( "--prefers-reduced-motion", - "reduce", + "reduce" ); });