Install any skill in seconds. Free to start, no credit card required.
Get Started Free →AI-enhanced visual testing with Playwright combining screenshot comparison, visual AI engines, and intelligent diff analysis for catching visual regressions at scale.
.claude/skills/pramoddutta-playwright-visual-ai-testing/SKILL.md| Test case | Without → With | Effect | Δ tokens | Δ turns |
|---|---|---|---|---|
| case-01 | ✗→✓ | ▲ Improved | 208% | 0% |
| case-08 | ✗→✓ | ▲ Improved | 215% | 0% |
| case-12 | ✗→✓ | ▲ Improved | 297% | 0% |
| case-14 | ✗→✓ | ▲ Improved | 381% | 0% |
| case-15 | ✗→✓ | ▲ Improved | 561% | 0% |
You are an expert QA engineer specializing in AI-enhanced visual testing with Playwright. When the user asks you to set up, write, review, or debug visual regression tests, visual AI integrations, or screenshot comparison pipelines, follow these detailed instructions. You understand baseline management, pixel-level comparison, AI-powered visual engines (Applitools Eyes, Percy, Chromatic), responsive visual testing, dynamic content masking, theme testing, and CI integration with visual review gates.
project-root/
├── src/
│ └── components/ # Application components
│
├── tests/
│ ├── visual/
│ │ ├── pages/
│ │ │ ├── homepage.visual.ts # Homepage visual tests
│ │ │ ├── dashboard.visual.ts # Dashboard visual tests
│ │ │ ├── settings.visual.ts # Settings page visual tests
│ │ │ └── auth.visual.ts # Auth pages visual tests
│ │ ├── components/
│ │ │ ├── button.visual.ts # Button component visual tests
│ │ │ ├── card.visual.ts # Card component visual tests
│ │ │ ├── navigation.visual.ts # Navigation visual tests
│ │ │ └── form.visual.ts # Form component visual tests
│ │ ├── responsive/
│ │ │ ├── mobile.visual.ts # Mobile viewport tests
│ │ │ ├── tablet.visual.ts # Tablet viewport tests
│ │ │ └── desktop.visual.ts # Desktop viewport tests
│ │ ├── themes/
│ │ │ ├── light-theme.visual.ts # Light theme tests
│ │ │ ├── dark-theme.visual.ts # Dark theme tests
│ │ │ └── high-contrast.visual.ts # High contrast tests
│ │ └── cross-browser/
│ │ └── browser-matrix.visual.ts # Cross-browser tests
│ ├── fixtures/
│ │ ├── visual.fixture.ts # Visual test fixtures
│ │ └── viewports.ts # Viewport definitions
│ ├── utils/
│ │ ├── screenshot.ts # Screenshot utilities
│ │ ├── masking.ts # Dynamic content masking
│ │ ├── baseline.ts # Baseline management
│ │ └── diff-reporter.ts # Custom diff reporter
│ └── __screenshots__/ # Baseline screenshots (git-tracked)
│ ├── chromium/
│ ├── firefox/
│ └── webkit/
│
├── playwright.config.ts
├── applitools.config.ts # Applitools Eyes config (optional)
└── package.jsontypescript// playwright.config.ts import { defineConfig, devices } from '@playwright/test'; export default defineConfig({ testDir: './tests/visual', testMatch: '**/*.visual.ts', timeout: 60_000, expect: { toHaveScreenshot: { maxDiffPixels: 100, // Allow up to 100 different pixels maxDiffPixelRatio: 0.01, // Or 1% of total pixels threshold: 0.2, // Per-pixel color difference threshold (0-1) animations: 'disabled', // Disable CSS animations for stable screenshots }, }, use: { trace: 'on-first-retry', screenshot: 'only-on-failure', actionTimeout: 10_000, }, // Visual test projects across browsers and viewports projects: [ // Desktop browsers { name: 'chromium-desktop', use: { ...devices['Desktop Chrome'], viewport: { width: 1920, height: 1080 }, }, }, { name: 'firefox-desktop', use: { ...devices['Desktop Firefox'], viewport: { width: 1920, height: 1080 }, }, }, { name: 'webkit-desktop', use: { ...devices['Desktop Safari'], viewport: { width: 1920, height: 1080 }, }, }, // Tablet { name: 'chromium-tablet', use: { ...devices['iPad (gen 7)'], }, }, // Mobile { name: 'chromium-mobile', use: { ...devices['iPhone 14'], }, }, { name: 'chromium-mobile-landscape', use: { ...devices['iPhone 14 landscape'], }, }, ], // Snapshot path template snapshotPathTemplate: '{testDir}/__screenshots__/{projectName}/{testFilePath}/{arg}{ext}', });
typescript// tests/fixtures/viewports.ts export const viewports = { mobile: { width: 375, height: 812 }, // iPhone 14 mobileLandscape: { width: 812, height: 375 }, tablet: { width: 768, height: 1024 }, // iPad tabletLandscape: { width: 1024, height: 768 }, laptop: { width: 1366, height: 768 }, desktop: { width: 1920, height: 1080 }, ultrawide: { width: 2560, height: 1440 }, } as const; export type ViewportName = keyof typeof viewports; export const breakpoints: ViewportName[] = ['mobile', 'tablet', 'laptop', 'desktop'];
typescript// tests/fixtures/visual.fixture.ts import { test as base, expect, type Page, type Locator } from '@playwright/test'; import { viewports, type ViewportName } from './viewports'; interface VisualFixtures { visualPage: Page; screenshotOptions: { fullPage: boolean; animations: 'disabled' | 'allow'; mask?: Locator[]; maxDiffPixelRatio?: number; }; } export const test = base.extend<VisualFixtures>({ visualPage: async ({ page }, use) => { // Disable animations globally for stable screenshots await page.addStyleTag({ content: ` *, *::before, *::after { animation-duration: 0s !important; animation-delay: 0s !important; transition-duration: 0s !important; transition-delay: 0s !important; caret-color: transparent !important; } `, }); await use(page); }, screenshotOptions: async ({}, use) => { await use({ fullPage: true, animations: 'disabled', maxDiffPixelRatio: 0.01, }); }, }); export { expect };
typescript// tests/utils/screenshot.ts import { type Page, type Locator, expect } from '@playwright/test'; /** * Take a full-page screenshot with standard visual test settings. */ export async function assertPageScreenshot( page: Page, name: string, options: { fullPage?: boolean; mask?: Locator[]; maxDiffPixelRatio?: number; threshold?: number; } = {}, ): Promise<void> { await page.waitForLoadState('networkidle'); await hideScrollbars(page); await waitForImages(page); await waitForFonts(page); await expect(page).toHaveScreenshot(`${name}.png`, { fullPage: options.fullPage ?? true, mask: options.mask ?? [], maxDiffPixelRatio: options.maxDiffPixelRatio ?? 0.01, threshold: options.threshold ?? 0.2, animations: 'disabled', }); } /** * Take a component-level screenshot. */ export async function assertComponentScreenshot( locator: Locator, name: string, options: { mask?: Locator[]; maxDiffPixelRatio?: number; padding?: number; } = {}, ): Promise<void> { await expect(locator).toHaveScreenshot(`${name}.png`, { maxDiffPixelRatio: options.maxDiffPixelRatio ?? 0.01, animations: 'disabled', }); } /** * Hide scrollbars for consistent screenshots. */ async function hideScrollbars(page: Page): Promise<void> { await page.addStyleTag({ content: ` ::-webkit-scrollbar { display: none !important; } * { scrollbar-width: none !important; } `, }); } /** * Wait for all images to load. */ async function waitForImages(page: Page): Promise<void> { await page.evaluate(() => { return Promise.all( Array.from(document.images) .filter((img) => !img.complete) .map( (img) => new Promise((resolve) => { img.onload = resolve; img.onerror = resolve; }), ), ); }); } /** * Wait for web fonts to load. */ async function waitForFonts(page: Page): Promise<void> { await page.evaluate(() => document.fonts.ready); }
typescript// tests/utils/masking.ts import { type Page, type Locator } from '@playwright/test'; /** * Get locators for common dynamic content that should be masked. */ export function getDynamicMasks(page: Page): Locator[] { return [ page.locator('[data-testid="timestamp"]'), page.locator('[data-testid="avatar"]'), page.locator('[data-testid="ad-slot"]'), page.locator('[data-testid="live-counter"]'), page.locator('time'), page.locator('.relative-time'), ]; } /** * Replace dynamic text content with static placeholders. */ export async function stabilizeDynamicContent(page: Page): Promise<void> { await page.evaluate(() => { // Replace all timestamps with a fixed value document.querySelectorAll('time, [data-testid="timestamp"]').forEach((el) => { el.textContent = 'Jan 1, 2024'; }); // Replace relative times document.querySelectorAll('.relative-time').forEach((el) => { el.textContent = '1 day ago'; }); // Replace user avatars with a placeholder document.querySelectorAll<HTMLImageElement>('[data-testid="avatar"]').forEach((img) => { img.src = 'data:image/svg+xml,<svg xmlns="http://www.w3.org/2000/svg" width="40" height="40"><rect fill="%23ccc" width="40" height="40"/></svg>'; }); // Freeze any counters or tickers document.querySelectorAll('[data-testid="live-counter"]').forEach((el) => { el.textContent = '42'; }); }); } /** * Hide elements that cause visual noise (tooltips, popovers, cursors). */ export async function hideVisualNoise(page: Page): Promise<void> { await page.addStyleTag({ content: ` [role="tooltip"], [data-radix-popper-content-wrapper], .tooltip, .popover { visibility: hidden !important; } * { cursor: none !important; caret-color: transparent !important; } `, }); }
typescript// tests/visual/pages/homepage.visual.ts import { test, expect } from '../../fixtures/visual.fixture'; import { assertPageScreenshot } from '../../utils/screenshot'; import { getDynamicMasks, stabilizeDynamicContent } from '../../utils/masking'; import { viewports, breakpoints } from '../../fixtures/viewports'; test.describe('Homepage Visual Tests', () => { test.beforeEach(async ({ page }) => { await page.goto('/'); await page.waitForLoadState('networkidle'); }); test('should match homepage baseline', async ({ page }) => { await stabilizeDynamicContent(page); await assertPageScreenshot(page, 'homepage-full', { mask: getDynamicMasks(page), }); }); test('should match hero section', async ({ page }) => { const hero = page.locator('[data-testid="hero-section"]'); await expect(hero).toHaveScreenshot('homepage-hero.png', { animations: 'disabled', }); }); test('should match navigation bar', async ({ page }) => { const nav = page.locator('nav'); await expect(nav).toHaveScreenshot('homepage-nav.png'); }); test('should match footer', async ({ page }) => { const footer = page.locator('footer'); await footer.scrollIntoViewIfNeeded(); await expect(footer).toHaveScreenshot('homepage-footer.png'); }); // Responsive visual tests for (const breakpoint of breakpoints) { test(`should match at ${breakpoint} viewport`, async ({ page }) => { await page.setViewportSize(viewports[breakpoint]); await stabilizeDynamicContent(page); await assertPageScreenshot(page, `homepage-${breakpoint}`, { mask: getDynamicMasks(page), }); }); } });
typescript// tests/visual/pages/dashboard.visual.ts import { test, expect } from '../../fixtures/visual.fixture'; import { assertPageScreenshot, assertComponentScreenshot } from '../../utils/screenshot'; import { getDynamicMasks, stabilizeDynamicContent } from '../../utils/masking'; test.describe('Dashboard Visual Tests', () => { test.beforeEach(async ({ page }) => { // Login and navigate to dashboard await page.goto('/login'); await page.fill('#email', 'test@example.com'); await page.fill('#password', 'password123'); await page.click('button[type="submit"]'); await page.waitForURL('/dashboard'); }); test('should match dashboard layout', async ({ page }) => { await stabilizeDynamicContent(page); await assertPageScreenshot(page, 'dashboard-full', { mask: getDynamicMasks(page), }); }); test('should match sidebar navigation', async ({ page }) => { const sidebar = page.locator('[data-testid="sidebar"]'); await assertComponentScreenshot(sidebar, 'dashboard-sidebar'); }); test('should match stats cards', async ({ page }) => { await stabilizeDynamicContent(page); const statsSection = page.locator('[data-testid="stats-cards"]'); await assertComponentScreenshot(statsSection, 'dashboard-stats'); }); test('should match dashboard with collapsed sidebar', async ({ page }) => { await page.click('[data-testid="sidebar-toggle"]'); await page.waitForTimeout(300); // Wait for collapse animation await assertPageScreenshot(page, 'dashboard-collapsed-sidebar'); }); test('should match empty state', async ({ page }) => { // Navigate to section with no data await page.goto('/dashboard/reports?filter=empty'); await page.waitForLoadState('networkidle'); const emptyState = page.locator('[data-testid="empty-state"]'); await assertComponentScreenshot(emptyState, 'dashboard-empty-state'); }); });
typescript// tests/visual/components/button.visual.ts import { test, expect } from '../../fixtures/visual.fixture'; test.describe('Button Component Visual Tests', () => { test.beforeEach(async ({ page }) => { await page.goto('/storybook/button'); // Or a dedicated visual test page }); test('should match primary button variants', async ({ page }) => { const variants = ['default', 'destructive', 'outline', 'secondary', 'ghost', 'link']; for (const variant of variants) { const button = page.locator(`[data-testid="button-${variant}"]`); await expect(button).toHaveScreenshot(`button-${variant}.png`); } }); test('should match button sizes', async ({ page }) => { const sizes = ['sm', 'default', 'lg', 'icon']; for (const size of sizes) { const button = page.locator(`[data-testid="button-size-${size}"]`); await expect(button).toHaveScreenshot(`button-size-${size}.png`); } }); test('should match button hover state', async ({ page }) => { const button = page.locator('[data-testid="button-default"]'); await button.hover(); await expect(button).toHaveScreenshot('button-hover.png'); }); test('should match button focus state', async ({ page }) => { const button = page.locator('[data-testid="button-default"]'); await button.focus(); await expect(button).toHaveScreenshot('button-focus.png'); }); test('should match button disabled state', async ({ page }) => { const button = page.locator('[data-testid="button-disabled"]'); await expect(button).toHaveScreenshot('button-disabled.png'); }); test('should match button with loading spinner', async ({ page }) => { const button = page.locator('[data-testid="button-loading"]'); // Freeze the spinner animation await page.addStyleTag({ content: '[data-testid="button-loading"] svg { animation: none !important; }', }); await expect(button).toHaveScreenshot('button-loading.png'); }); });
typescript// tests/visual/components/form.visual.ts import { test, expect } from '../../fixtures/visual.fixture'; test.describe('Form Component Visual Tests', () => { test.beforeEach(async ({ page }) => { await page.goto('/storybook/form'); }); test('should match input default state', async ({ page }) => { const input = page.locator('[data-testid="input-default"]'); await expect(input).toHaveScreenshot('input-default.png'); }); test('should match input with value', async ({ page }) => { const input = page.locator('[data-testid="input-default"] input'); await input.fill('John Doe'); await input.blur(); await expect(page.locator('[data-testid="input-default"]')).toHaveScreenshot( 'input-with-value.png', ); }); test('should match input error state', async ({ page }) => { const input = page.locator('[data-testid="input-error"]'); await expect(input).toHaveScreenshot('input-error.png'); }); test('should match select dropdown', async ({ page }) => { const select = page.locator('[data-testid="select-default"]'); await select.click(); await page.waitForTimeout(100); // Screenshot includes the open dropdown await expect(page).toHaveScreenshot('select-open.png', { fullPage: false, }); }); test('should match complete form layout', async ({ page }) => { const form = page.locator('[data-testid="complete-form"]'); await expect(form).toHaveScreenshot('form-complete.png'); }); });
typescript// tests/visual/themes/dark-theme.visual.ts import { test, expect } from '../../fixtures/visual.fixture'; import { assertPageScreenshot } from '../../utils/screenshot'; import { getDynamicMasks, stabilizeDynamicContent } from '../../utils/masking'; test.describe('Dark Theme Visual Tests', () => { test.beforeEach(async ({ page }) => { // Set dark theme via cookie/localStorage/class await page.addInitScript(() => { localStorage.setItem('theme', 'dark'); document.documentElement.classList.add('dark'); }); await page.goto('/'); await page.waitForLoadState('networkidle'); }); test('should match homepage in dark theme', async ({ page }) => { await stabilizeDynamicContent(page); await assertPageScreenshot(page, 'homepage-dark', { mask: getDynamicMasks(page), }); }); test('should match navigation in dark theme', async ({ page }) => { const nav = page.locator('nav'); await expect(nav).toHaveScreenshot('nav-dark.png'); }); test('should match cards in dark theme', async ({ page }) => { const card = page.locator('[data-testid="card"]').first(); await expect(card).toHaveScreenshot('card-dark.png'); }); test('should match form inputs in dark theme', async ({ page }) => { await page.goto('/login'); const form = page.locator('form'); await expect(form).toHaveScreenshot('login-form-dark.png'); }); test('should match dashboard in dark theme', async ({ page }) => { await page.goto('/login'); await page.fill('#email', 'test@example.com'); await page.fill('#password', 'password123'); await page.click('button[type="submit"]'); await page.waitForURL('/dashboard'); await stabilizeDynamicContent(page); await assertPageScreenshot(page, 'dashboard-dark', { mask: getDynamicMasks(page), }); }); });
typescript// tests/visual/themes/light-theme.visual.ts import { test, expect } from '../../fixtures/visual.fixture'; import { assertPageScreenshot } from '../../utils/screenshot'; import { getDynamicMasks, stabilizeDynamicContent } from '../../utils/masking'; test.describe('Light Theme Visual Tests', () => { test.beforeEach(async ({ page }) => { await page.addInitScript(() => { localStorage.setItem('theme', 'light'); document.documentElement.classList.remove('dark'); }); await page.goto('/'); await page.waitForLoadState('networkidle'); }); test('should match homepage in light theme', async ({ page }) => { await stabilizeDynamicContent(page); await assertPageScreenshot(page, 'homepage-light', { mask: getDynamicMasks(page), }); }); test('should match contrast between themes', async ({ page }) => { // Take light screenshot await expect(page).toHaveScreenshot('theme-comparison-light.png', { fullPage: false, }); // Switch to dark await page.evaluate(() => { document.documentElement.classList.add('dark'); localStorage.setItem('theme', 'dark'); }); await page.waitForTimeout(100); await expect(page).toHaveScreenshot('theme-comparison-dark.png', { fullPage: false, }); }); });
typescript// tests/visual/cross-browser/browser-matrix.visual.ts import { test, expect } from '@playwright/test'; import { assertPageScreenshot } from '../../utils/screenshot'; import { stabilizeDynamicContent, getDynamicMasks } from '../../utils/masking'; test.describe('Cross-Browser Visual Consistency', () => { const criticalPages = ['/', '/login', '/pricing', '/about']; for (const pagePath of criticalPages) { test(`should render ${pagePath} consistently`, async ({ page, browserName }) => { await page.goto(pagePath); await page.waitForLoadState('networkidle'); await stabilizeDynamicContent(page); const pageName = pagePath === '/' ? 'homepage' : pagePath.slice(1); await assertPageScreenshot(page, `${pageName}-${browserName}`, { mask: getDynamicMasks(page), // Higher tolerance for cross-browser due to rendering engine differences maxDiffPixelRatio: 0.02, threshold: 0.3, }); }); } });
typescript// applitools.config.ts import { type EyesConfig } from '@applitools/eyes-playwright'; export const config: EyesConfig = { apiKey: process.env.APPLITOOLS_API_KEY!, appName: 'My Application', matchLevel: 'Layout', // 'Strict' | 'Layout' | 'Content' | 'Exact' batch: { name: `Visual Tests - ${process.env.CI ? 'CI' : 'Local'}`, id: process.env.GITHUB_RUN_ID || `local-${Date.now()}`, }, browser: [ { width: 1920, height: 1080, name: 'chrome' }, { width: 1920, height: 1080, name: 'firefox' }, { width: 1920, height: 1080, name: 'safari' }, { width: 768, height: 1024, name: 'chrome' }, { width: 375, height: 812, name: 'chrome' }, ], accessibilityValidation: { level: 'AA', guidelinesVersion: 'WCAG_2_1', }, };
typescript// tests/visual/applitools-example.visual.ts import { test } from '@playwright/test'; import { Eyes, Target, VisualGridRunner, Configuration } from '@applitools/eyes-playwright'; let runner: VisualGridRunner; let eyes: Eyes; test.beforeAll(() => { runner = new VisualGridRunner({ testConcurrency: 5 }); }); test.beforeEach(async ({ page }) => { eyes = new Eyes(runner); const configuration = new Configuration(); configuration.setApiKey(process.env.APPLITOOLS_API_KEY!); configuration.setAppName('My Application'); configuration.setBatch({ name: 'Visual AI Tests', id: process.env.GITHUB_RUN_ID, }); // Render on multiple browsers via Ultrafast Grid configuration.addBrowser(1920, 1080, 'chrome'); configuration.addBrowser(1920, 1080, 'firefox'); configuration.addBrowser(768, 1024, 'chrome'); configuration.addDeviceEmulation('iPhone 14'); eyes.setConfiguration(configuration); await eyes.open(page, 'My Application', test.info().title); }); test.afterEach(async () => { await eyes.close(false); }); test.afterAll(async () => { const results = await runner.getAllTestResults(false); console.log('Applitools results:', results.toString()); }); test('homepage visual AI check', async ({ page }) => { await page.goto('/'); // Full page check with AI await eyes.check('Homepage', Target.window().fully()); // Region check await eyes.check( 'Hero Section', Target.region('[data-testid="hero-section"]'), ); // Layout match level for responsive components await eyes.check( 'Navigation', Target.region('nav').layout(), ); }); test('dashboard visual AI check', async ({ page }) => { await page.goto('/dashboard'); // Ignore dynamic regions await eyes.check( 'Dashboard', Target.window() .fully() .ignoreRegions('[data-testid="timestamp"]', '[data-testid="live-counter"]'), ); // Strict check for critical UI await eyes.check( 'Stats Cards', Target.region('[data-testid="stats-cards"]').strict(), ); });
typescript// tests/visual/percy-example.visual.ts import { test } from '@playwright/test'; import percySnapshot from '@percy/playwright'; test.describe('Percy Visual Tests', () => { test('should capture homepage', async ({ page }) => { await page.goto('/'); await page.waitForLoadState('networkidle'); await percySnapshot(page, 'Homepage', { widths: [375, 768, 1280, 1920], minHeight: 1024, percyCSS: ` [data-testid="timestamp"] { visibility: hidden; } [data-testid="avatar"] { visibility: hidden; } `, }); }); test('should capture dashboard states', async ({ page }) => { await page.goto('/dashboard'); // Default state await percySnapshot(page, 'Dashboard - Default'); // With sidebar collapsed await page.click('[data-testid="sidebar-toggle"]'); await percySnapshot(page, 'Dashboard - Sidebar Collapsed'); // With modal open await page.click('[data-testid="create-button"]'); await page.waitForSelector('[role="dialog"]'); await percySnapshot(page, 'Dashboard - Create Modal'); }); test('should capture form validation states', async ({ page }) => { await page.goto('/login'); // Empty form await percySnapshot(page, 'Login - Empty'); // With validation errors await page.click('button[type="submit"]'); await page.waitForSelector('.error-message'); await percySnapshot(page, 'Login - Validation Errors'); }); });
typescript// tests/utils/baseline.ts import { execSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; const SCREENSHOTS_DIR = path.resolve(__dirname, '../__screenshots__'); /** * Update all visual baselines (run after intentional visual changes). * Usage: npx playwright test --update-snapshots */ export function getBaselineInfo(): { totalBaselines: number; lastUpdated: string; browsers: string[]; } { if (!fs.existsSync(SCREENSHOTS_DIR)) { return { totalBaselines: 0, lastUpdated: 'never', browsers: [] }; } const browsers = fs .readdirSync(SCREENSHOTS_DIR) .filter((f) => fs.statSync(path.join(SCREENSHOTS_DIR, f)).isDirectory()); let totalBaselines = 0; for (const browser of browsers) { const browserDir = path.join(SCREENSHOTS_DIR, browser); const files = fs.readdirSync(browserDir).filter((f) => f.endsWith('.png')); totalBaselines += files.length; } const lastUpdated = execSync('git log -1 --format=%ci -- tests/__screenshots__/') .toString() .trim(); return { totalBaselines, lastUpdated, browsers }; } /** * List all baseline files that have changed since the base branch. */ export function getChangedBaselines(baseBranch: string = 'main'): string[] { try { const output = execSync( `git diff --name-only ${baseBranch}...HEAD -- tests/__screenshots__/`, ).toString(); return output.split('\n').filter(Boolean); } catch { return []; } }
yaml# .github/workflows/visual-tests.yml name: Visual Regression Tests on: pull_request: branches: [main] jobs: visual-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 - run: npm ci - run: npx playwright install --with-deps # Run visual tests - run: npx playwright test tests/visual/ env: APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }} PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }} # Upload screenshot diffs as artifacts - uses: actions/upload-artifact@v4 if: failure() with: name: visual-diffs path: | test-results/ playwright-report/ # Post screenshot diff summary as PR comment - uses: actions/github-script@v7 if: failure() with: script: | const fs = require('fs'); const diffDir = 'test-results'; let comment = '## Visual Regression Report\n\n'; comment += 'Visual differences were detected. Please review the artifacts.\n'; await github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: comment, });
bash# Run all visual tests npx playwright test tests/visual/ # Update baselines after intentional changes npx playwright test tests/visual/ --update-snapshots # Run visual tests for a specific browser npx playwright test tests/visual/ --project=chromium-desktop # Run only theme tests npx playwright test tests/visual/themes/ # Run responsive tests npx playwright test tests/visual/responsive/ # View visual diff report npx playwright show-report # Run with trace for debugging npx playwright test tests/visual/ --trace on
waitForLoadState('networkidle') and wait for document.fonts.ready.__screenshots__/ to the repository so visual changes are code-reviewed alongside code changes.maxDiffPixelRatio: 0.01 catches real regressions while tolerating sub-pixel rendering differences. Adjust per platform.threshold (0.2-0.3) to tolerate anti-aliasing variations without missing real color changes.| Case | Status | Duration (ms) | Turns | Tokens | Tool calls | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Without | With | Δ | Without | With | Δ | Without | With | Δ | Without | With | Δ | ||
case-01 | fail→pass | 25,112 | 22,765 | -9% | 1 | 1 | 0% | 4,435 | 13,660 | +208% | 0 | 0 | — |
case-02 | fail→fail | 44,904 | 24,546 | -45% | 1 | 1 | 0% | 5,535 | 14,178 | +156% | 0 | 0 | — |
case-03 | fail→fail | 21,036 | 20,611 | -2% | 1 | 1 | 0% | 4,017 | 12,681 | +216% | 0 | 0 | — |
case-04 | fail→fail | 13,219 | 14,071 | +6% | 1 | 1 | 0% | 2,256 | 11,667 | +417% | 0 | 0 | — |
case-05 | pass→pass | 12,919 | 19,506 | +51% | 1 | 1 | 0% | 2,373 | 12,948 | +446% | 0 | 0 | — |
case-06 | pass→pass | 8,350 | 10,862 | +30% | 1 | 1 | 0% | 1,442 | 11,015 | +664% | 0 | 0 | — |
case-07 | pass→pass | 21,106 | 15,423 | -27% | 1 | 1 | 0% | 3,700 | 11,795 | +219% | 0 | 0 | — |
case-08 | fail→pass | 22,117 | 16,617 | -25% | 1 | 1 | 0% | 3,801 | 11,974 | +215% | 0 | 0 | — |
case-09 | pass→pass | 17,966 | 24,315 | +35% | 1 | 1 | 0% | 3,043 | 13,322 | +338% | 0 | 0 | — |
case-10 | pass→pass | 27,723 | 24,420 | -12% | 1 | 1 | 0% | 4,439 | 14,045 | +216% | 0 | 0 | — |
case-11 | pass→pass | 41,581 | 17,688 | -57% | 1 | 1 | 0% | 2,826 | 12,297 | +335% | 0 | 0 | — |
case-12 | fail→pass | 19,030 | 18,491 | -3% | 1 | 1 | 0% | 3,134 | 12,440 | +297% | 0 | 0 | — |
case-13 | fail→fail | 20,474 | 27,467 | +34% | 1 | 1 | 0% | 3,495 | 12,824 | +267% | 0 | 0 | — |
case-14 | fail→pass | 25,957 | 14,123 | -46% | 1 | 1 | 0% | 2,417 | 11,629 | +381% | 0 | 0 | — |
case-15 | fail→pass | 12,882 | 12,924 | +0% | 1 | 1 | 0% | 1,715 | 11,331 | +561% | 0 | 0 | — |
case-16 | pass→pass | 39,075 | 16,544 | -58% | 1 | 1 | 0% | 3,019 | 12,038 | +299% | 0 | 0 | — |
case-17 | fail→pass | 22,772 | 20,309 | -11% | 1 | 1 | 0% | 3,041 | 12,198 | +301% | 0 | 0 | — |
case-18 | fail→pass | 20,679 | 42,628 | +106% | 1 | 1 | 0% | 3,052 | 11,702 | +283% | 0 | 0 | — |
case-19 | pass→fail | 16,747 | 19,431 | +16% | 1 | 1 | 0% | 2,685 | 12,989 | +384% | 0 | 0 | — |
case-20 | pass→pass | 16,917 | 19,762 | +17% | 1 | 1 | 0% | 3,147 | 11,517 | +266% | 0 | 0 | — |
case-21 | pass→pass | 12,397 | 11,094 | -11% | 1 | 1 | 0% | 1,876 | 11,385 | +507% | 0 | 0 | — |
case-22 | fail→pass | 20,470 | 12,464 | -39% | 1 | 1 | 0% | 3,452 | 11,204 | +225% | 0 | 0 | — |
DecimalAI ran this skill against gemini-3.6-flash twice over the same eval suite — once with the skill loaded and once without — and compared the two runs case by case. 22 cases were attempted. The headline lift of +32 percentage points is the difference between those two pass rates over the 22 comparable cases. 1 case got worse with the skill loaded, and it is included in that figure.
Without the skill loaded, the model failed this case. With it loaded, the same prompt on the same model passed. This is one improved case from the latest verified run; every case, including any that regressed, is in the table above.
Other measured skills in the registry, with their headline benchmark lift.