004-agent-setup-nextjs

Prompt

Setup this project to work with Primer. Including a default page layout in app.tsx.

Checks

• node-tests

  vitest.config.scenario.ts

  import {defineConfig} from 'vitest/config'
  
  export default defineConfig({
    test: {
      include: ['scenario.test.ts'],
      reporters: [
        [
          'json',
          {
            outputFile: 'vitest-scenario-report.json',
            includeTaskLocation: true,
          },
        ],
      ],
    },
  })
  

  scenario.test.ts

  import fs from 'node:fs/promises'
  import path from 'node:path'
  import {expect, test} from 'vitest'
  
  const packageJsonPath = path.resolve(import.meta.dirname, 'package.json')
  const layoutPath = path.resolve(import.meta.dirname, 'src', 'app', 'layout.tsx')
  const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8'))
  const layout = await fs.readFile(layoutPath, 'utf8')
  const dependencies = {
    ...packageJson.dependencies,
    ...packageJson.devDependencies,
  }
  
  test('includes @primer/react', () => {
    expect(dependencies).toHaveProperty('@primer/react')
  })
  
  test('includes @primer/primitives', () => {
    expect(dependencies).toHaveProperty('@primer/primitives')
  })
  
  test('src/app/layout.tsx imports Primer BaseStyles', () => {
    expect(layout).toMatch(/import\s+{[^}]*\bBaseStyles\b[^}]*}\s+from\s+['"]@primer\/react['"]/)
  })
  
  test('src/app/layout.tsx uses Primer BaseStyles', () => {
    expect(layout).toMatch(/<BaseStyles(?:\s[^>]*)?>[\s\S]*?<\/BaseStyles>/)
  })
  
  test('src/app/layout.tsx configures automatic light and dark color modes', () => {
    const html = layout.match(/<html\b[^>]*>/)?.[0]
  
    expect(html).toBeDefined()
    expect(html).toMatch(/\bdata-color-mode=["']auto["']/)
    expect(html).toMatch(/\bdata-light-theme=["']light["']/)
    expect(html).toMatch(/\bdata-dark-theme=["']dark["']/)
  })
  
  test('src/app/layout.tsx imports the light color mode primitives', () => {
    expect(layout).toMatch(/import\s+['"]@primer\/primitives\/dist\/css\/functional\/themes\/light\.css['"]/)
  })
  
  test('src/app/layout.tsx imports the dark color mode primitives', () => {
    expect(layout).toMatch(/import\s+['"]@primer\/primitives\/dist\/css\/functional\/themes\/dark\.css['"]/)
  })
  
  test('src/app/layout.tsx does not import ThemeProvider', () => {
    expect(layout).not.toMatch(/\bThemeProvider\b/)
  })
  

Scenario configuration

import {defineConfig} from '@primer/agent-eval/scenario'
import type {JsonTestResults} from 'vitest/node'

export default defineConfig({
  description: 'Evaluate whether the agent configures a Next.js project to use Primer.',
  prompt: `Setup this project to work with Primer. Including a default page layout in app.tsx.`,
  tags: ['baseline', 'nextjs', 'primer', 'setup'],
  checks: [
    {
      name: 'node-tests',
      files: ['vitest.config.scenario.ts', 'scenario.test.ts'],
      async run({sandbox}) {
        const commandResult = await sandbox.runCommand(
          'npx',
          ['vitest', 'run', '--config', 'vitest.config.scenario.ts'],
          {allowNonZeroExitCode: true},
        )
        if (commandResult.exitCode !== 0 && commandResult.exitCode !== 1) {
          throw new Error(`Vitest failed with exit code ${commandResult.exitCode}: ${commandResult.stderr}`)
        }
        const contents = await sandbox.readFile('vitest-scenario-report.json')
        const json: JsonTestResults = JSON.parse(contents)
        if (commandResult.exitCode !== 0 && json.numFailedTests === 0) {
          throw new Error(`Vitest failed without reporting failed tests: ${commandResult.stderr}`)
        }
        return {
          outcomes: json.testResults.flatMap(({assertionResults}) => {
            return assertionResults.map(assertion => {
              return {
                type: 'outcome',
                id: assertion.fullName,
                status: assertion.status === 'passed' ? 'passed' : assertion.status === 'failed' ? 'failed' : 'skipped',
              }
            })
          }),
        }
      },
    },
  ],
})

Experiments

This scenario is not included in any experiments.