Auditing in CI

name: Accessibility Audit

on:
  pull_request:

jobs:
  audit:
    name: Audit pages
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: "1.3.5"

      - uses: actions/setup-node@v4
        with:
          node-version: "20"

      - name: Install dependencies
        run: bun install --frozen-lockfile

      - name: Build
        run: bun run build
        env:
          NEXT_PUBLIC_SITE_URL: http://localhost:3000
          # Add any other env vars your production build requires.

      - name: Install Playwright browsers
        run: npx playwright install --with-deps chromium

      - name: Start production server
        run: npx next start --port 3000 &

      - name: Wait for server
        run: npx wait-on http://127.0.0.1:3000 --timeout 60000

      - name: Run accessibility audit
        run: npx tsx scripts/a11y-audit.ts / /docs /blog
        env:
          AUDIT_SUMMARY_PATH: audit-results.md
          AUDIT_TAGS: A,AA

      - name: Write job summary
        if: always()
        run: |
          if [ -f audit-results.md ]; then
            cat audit-results.md >> "$GITHUB_STEP_SUMMARY"
          fi

{
  "scripts": {
    "audit": "tsx scripts/a11y-audit.ts / /docs /blog /pricing"
  },
  "devDependencies": {
    "@access-kit/react": "0.1.0",
    "esbuild": "0.25.0",
    "playwright": "1.52.0",
    "tsx": "4.19.0",
    "wait-on": "9.0.5"
  }
}

/**
 * Headless accessibility audit using @access-kit/react's runAccessibilityAudit + Playwright.
 *
 * Prerequisites:
 *   - A built app and a running server at AUDIT_BASE_URL (any framework).
 *   - Dependencies: @access-kit/react, playwright, esbuild, tsx (or node with the script compiled).
 *
 * Usage:
 *   npx tsx scripts/a11y-audit.ts / /about /pricing
 *   AUDIT_PAGES="/,/about" npx tsx scripts/a11y-audit.ts
 *
 * Environment:
 *   AUDIT_BASE_URL     — origin only, default http://localhost:3000
 *   AUDIT_PAGES        — comma-separated paths if no CLI args (e.g. /,/docs,/blog)
 *   AUDIT_TAGS         — comma-separated blocking tags, default A,AA (see README)
 *   AUDIT_SUMMARY_PATH — optional path to write a markdown report (e.g. for CI summaries)
 */

import { chromium, type BrowserContext } from "playwright"
import { build } from "esbuild"
import {
  formatAuditFindingsForConsole,
  type AccessKitAuditFinding,
} from "@access-kit/react"
import { writeFileSync } from "fs"

//=============== Configuration ===============

const BASE_URL = process.env.AUDIT_BASE_URL ?? "http://localhost:3000"
const SUMMARY_PATH = process.env.AUDIT_SUMMARY_PATH

function parsePagesFromArgv(): string[] {
  return process.argv.slice(2).map((page) => page.trim()).filter(Boolean)
}

function parsePagesFromEnv(): string[] {
  const raw = process.env.AUDIT_PAGES
  if (!raw) return []
  return raw
    .split(",")
    .map((page) => page.trim())
    .filter(Boolean)
}

function normalizePath(path: string): string {
  if (!path.startsWith("/")) return `/${path}`
  return path
}

function resolvePages(): string[] {
  const fromArgv = parsePagesFromArgv()
  if (fromArgv.length > 0) return fromArgv.map(normalizePath)
  const fromEnv = parsePagesFromEnv()
  if (fromEnv.length > 0) return fromEnv.map(normalizePath)
  return []
}

// =============== Audit tags ===============
// Comma-separated list of tags that determine which findings block merges.
//   WCAG levels: A, AA, AAA
//   Additional:  best-practice, section508, ACT, cat.color, cat.forms, …
// Findings matching a listed tag are blocking; the rest are still reported
// but won't cause a non-zero exit code.
// Additional (non-WCAG-level) tags are also forwarded to runAccessibilityAudit
// so those extra rules actually run during the scan.

const WCAG_LEVELS = new Set(["A", "AA", "AAA"])

function parseAuditTags(value: string | undefined): string[] {
  if (!value) return ["A", "AA"]
  return value
    .split(",")
    .map((tag) => tag.trim())
    .filter(Boolean)
}

const AUDIT_TAGS = parseAuditTags(process.env.AUDIT_TAGS)

const BLOCKING_LEVELS = new Set(
  AUDIT_TAGS.filter((tag) => WCAG_LEVELS.has(tag.toUpperCase())).map((tag) =>
    tag.toUpperCase(),
  ),
)

const ADDITIONAL_TAGS = AUDIT_TAGS.filter(
  (tag) => !WCAG_LEVELS.has(tag.toUpperCase()),
)

const BLOCKING_CATEGORIES = new Set(
  ADDITIONAL_TAGS.filter((tag) => tag.startsWith("cat.")).map((tag) =>
    tag.replace(/^cat\./i, "").toLowerCase(),
  ),
)

const HAS_STANDARD_ADDITIONAL = ADDITIONAL_TAGS.some(
  (tag) => !tag.startsWith("cat."),
)

function isBlockingError(finding: AccessKitAuditFinding): boolean {
  if (finding.severity !== "error") return false
  if (finding.wcagLevel && BLOCKING_LEVELS.has(finding.wcagLevel)) return true
  if (finding.category && BLOCKING_CATEGORIES.has(finding.category)) return true
  if (!finding.wcagLevel && HAS_STANDARD_ADDITIONAL) return true
  return false
}

function countFindingsBySeverity(findings: AccessKitAuditFinding[]): {
  blocking: number
  nonBlocking: number
  warnings: number
} {
  const warnings = findings.filter((f) => f.severity === "warning").length
  const errors = findings.filter((f) => f.severity === "error")
  const blocking = errors.filter(isBlockingError).length
  return {
    blocking,
    nonBlocking: errors.length - blocking,
    warnings,
  }
}

const TAG_LABEL = AUDIT_TAGS.join(", ")

// =============== Audit bundle ===============
// Uses esbuild to create a browser-injectable IIFE from @access-kit/react.
// Tree-shaking ensures only runAccessibilityAudit + axe-core are included.

async function buildAuditBundle(): Promise<string> {
  const result = await build({
    stdin: {
      contents: [
        `import { runAccessibilityAudit } from "@access-kit/react";`,
        `globalThis.__runAccessKitAudit = (opts) => runAccessibilityAudit(document, opts);`,
      ].join("\n"),
      resolveDir: process.cwd(),
      loader: "ts",
    },
    bundle: true,
    format: "iife",
    write: false,
    platform: "browser",
    treeShaking: true,
    logLevel: "silent",
  })
  return result.outputFiles![0].text
}

// =============== Markdown summary ===============

function severityIcon(
  finding: AccessKitAuditFinding,
  isBlocking: boolean,
): string {
  if (finding.severity === "warning") return "🟡"
  if (isBlocking) return "🔴"
  return "⚪"
}

function truncateSelector(selector: string, maxLength: number): string {
  if (selector.length <= maxLength) return selector
  return `${selector.slice(0, maxLength - 3)}...`
}

function markdownFindingRow(finding: AccessKitAuditFinding): string {
  const blocking = isBlockingError(finding)
  const icon = severityIcon(finding, blocking)
  const level = finding.wcagLevel ?? "—"
  const selector = truncateSelector(finding.selector, 50)
  return `| ${icon} | ${finding.ruleId} | ${level} | ${finding.message} | \`${selector}\` |`
}

function generateMarkdownSummary(
  results: { page: string; findings: AccessKitAuditFinding[] }[],
  blockingErrors: number,
  nonBlockingErrors: number,
  totalWarnings: number,
): string {
  const lines: string[] = [
    "## AccessKit Accessibility Audit",
    "",
    `> Audit tags: **${TAG_LABEL}** — errors matching these tags block the build.`,
    "",
    "| Metric | Count |",
    "|---|---|",
    `| Pages scanned | ${results.length} |`,
    `| Blocking errors | ${blockingErrors} |`,
    `| Non-blocking errors | ${nonBlockingErrors} |`,
    `| Warnings | ${totalWarnings} |`,
    "",
  ]

  const allClean =
    blockingErrors === 0 && nonBlockingErrors === 0 && totalWarnings === 0
  if (allClean) {
    lines.push("> All pages passed the accessibility audit.")
    return lines.join("\n")
  }

    const pageSections = results
    .map(({ page, findings }) => ({
      page,
      blocking: findings.filter(isBlockingError),
    }))
    .filter((r) => r.blocking.length > 0)
    .flatMap(({ page, blocking }) => {
      return [
        `### \`${page}\`: ${blocking.length} blocking error(s)`,
        "",
        "| | Rule | Level | Standards | Message | Selector |",
        "|---|---|---|---|---|---|",
        ...blocking.map(markdownFindingRow),
        "",
      ]
    })

  lines.push(...pageSections)
  return lines.join("\n")
}

function printUsage(): void {
  console.error(`
AccessKit headless audit — run a production server first, then:

  npx tsx scripts/a11y-audit.ts / /about /pricing

Or set AUDIT_PAGES (comma-separated paths):

  AUDIT_PAGES="/,/about" npx tsx scripts/a11y-audit.ts

Environment: AUDIT_BASE_URL, AUDIT_TAGS, AUDIT_SUMMARY_PATH
`)
}

function logPageAuditResult(
  blocking: number,
): void {
  if (blocking > 0) {
    console.log(`✗ ${blocking} blocking error(s)`)
  } else {
    console.log("✓ pass")
  }
}

function printDetailedFindings(
  results: { page: string; findings: AccessKitAuditFinding[] }[],
): void {
  if (results.length === 0) return
  console.log("\n" + "━".repeat(60))
  results.forEach(({ page: pagePath, findings }) => {
    console.log(`\nPage: ${pagePath}`)
    console.log(formatAuditFindingsForConsole(findings))
  })
}

type AuditAccumulator = {
  allResults: { page: string; findings: AccessKitAuditFinding[] }[]
  blockingErrors: number
  nonBlockingErrors: number
  totalWarnings: number
}

const emptyAccumulator = (): AuditAccumulator => ({
  allResults: [],
  blockingErrors: 0,
  nonBlockingErrors: 0,
  totalWarnings: 0,
})

async function auditSinglePage(
  context: BrowserContext,
  bundle: string,
  pagePath: string,
  pad: number,
): Promise<{
  result: { page: string; findings: AccessKitAuditFinding[] }
  delta: Omit<AuditAccumulator, "allResults">
}> {
  const url = `${BASE_URL}${pagePath}`
  process.stdout.write(`  ${pagePath.padEnd(pad)} `)

  const page = await context.newPage()
  try {
    await page.goto(url, { waitUntil: "networkidle", timeout: 30_000 })
    await page.addScriptTag({ content: bundle })
    const findings: AccessKitAuditFinding[] = await page.evaluate(
      (tags) =>
        (globalThis as any).__runAccessKitAudit(
          tags.length > 0 ? { additionalTags: tags } : undefined,
        ),
      ADDITIONAL_TAGS,
    )

    const { blocking, nonBlocking, warnings } =
      countFindingsBySeverity(findings)
    logPageAuditResult(blocking)

    return {
      result: { page: pagePath, findings },
      delta: { blockingErrors: blocking, nonBlockingErrors: nonBlocking, totalWarnings: warnings },
    }
  } catch (err: unknown) {
    const message =
      err instanceof Error ? err.message.slice(0, 80) : String(err)
    console.log(`✗ error: ${message}`)
    return {
      result: { page: pagePath, findings: [] },
      delta: { blockingErrors: 0, nonBlockingErrors: 0, totalWarnings: 0 },
    }
  } finally {
    await page.close()
  }
}

async function runAuditsSequentially(
  pagePaths: string[],
  context: BrowserContext,
  bundle: string,
  pad: number,
): Promise<AuditAccumulator> {
  return pagePaths.reduce(
    async (accPromise, pagePath) => {
      const acc = await accPromise
      const { result, delta } = await auditSinglePage(
        context,
        bundle,
        pagePath,
        pad,
      )
      return {
        allResults: [...acc.allResults, result],
        blockingErrors: acc.blockingErrors + delta.blockingErrors,
        nonBlockingErrors: acc.nonBlockingErrors + delta.nonBlockingErrors,
        totalWarnings: acc.totalWarnings + delta.totalWarnings,
      }
    },
    Promise.resolve(emptyAccumulator()),
  )
}

// =============== Main ===============

async function main() {
  const PAGES = resolvePages()
  if (PAGES.length === 0) {
    printUsage()
    process.exit(1)
  }

  console.log("\n━━━ AccessKit Accessibility Audit ━━━\n")
  console.log(`Base URL: ${BASE_URL}`)
  console.log(`Pages:    ${PAGES.join(", ")}\n`)

  console.log("Bundling @access-kit/react for browser injection...")
  const bundle = await buildAuditBundle()
  console.log(`Bundle ready (${Math.round(bundle.length / 1024)} KB)\n`)

  console.log(`Audit tags: ${TAG_LABEL} (set via AUDIT_TAGS)\n`)

  const browser = await chromium.launch()
  const context = await browser.newContext()
  const pad = Math.max(...PAGES.map((p) => p.length), 4)

  try {
    const {
      allResults,
      blockingErrors,
      nonBlockingErrors,
      totalWarnings,
    } = await runAuditsSequentially(PAGES, context, bundle, pad)

    const blockingOnly = allResults
      .map((r) => ({
        page: r.page,
        findings: r.findings.filter(isBlockingError),
      }))
      .filter((r) => r.findings.length > 0)
    printDetailedFindings(blockingOnly)

    console.log("━".repeat(60))
    console.log(`\n  Audit tags:      ${TAG_LABEL}`)
    console.log(`  Pages scanned:     ${PAGES.length}`)
    console.log(`  Blocking errors:   ${blockingErrors}`)
    console.log(`  Non-blocking:      ${nonBlockingErrors}`)
    console.log(`  Warnings:          ${totalWarnings}`)

    if (SUMMARY_PATH) {
      const md = generateMarkdownSummary(
        allResults,
        blockingErrors,
        nonBlockingErrors,
        totalWarnings,
      )
      writeFileSync(SUMMARY_PATH, md, "utf-8")
      console.log(`  Summary written:   ${SUMMARY_PATH}`)
    }

    console.log("")
    if (blockingErrors > 0) {
      console.log(
        `Audit FAILED — ${blockingErrors} blocking error(s) for tags [${TAG_LABEL}].\n`,
      )
      process.exit(1)
    }
    console.log("Audit passed.\n")
  } finally {
    await context.close()
    await browser.close()
  }
}

main().catch((err) => {
  console.error("Fatal:", err)
  process.exit(1)
})

# After your server is up (e.g. http://localhost:3000)
npx tsx scripts/a11y-audit.ts / /about /pricing

# Or:
AUDIT_PAGES="/,/about,/pricing" npx tsx scripts/a11y-audit.ts