000-checks

Prompt

Example prompt

Checks

• node-tests

  Example for node-based tests with vitest

  vitest.config.scenario.ts

  import {defineConfig} from 'vitest/config'
  
  export default defineConfig({
    test: {
      include: ['scenario.test.ts'],
      reporters: [
        [
          'json',
          {
            outputFile: 'vitest-scenario-report.json',
            includeTaskLocation: true,
          },
        ],
      ],
    },
  })
  

  scenario.test.ts

  import fs from 'node:fs/promises'
  import path from 'node:path'
  import {expect, test} from 'vitest'
  
  const appPath = path.resolve(import.meta.dirname, 'src', 'App.tsx')
  const app = await fs.readFile(appPath, 'utf8')
  
  test('src/App.tsx exports the example app', () => {
    expect(app).toMatch(/export function App/)
  })
  

• browser-tests

  Example for browser-based tests with vitest

  vitest.config.browser.scenario.ts

  import {playwright} from '@vitest/browser-playwright'
  import {defineConfig} from 'vitest/config'
  
  export default defineConfig({
    test: {
      include: ['scenario.browser.test.ts'],
      browser: {
        enabled: true,
        headless: true,
        instances: [
          {
            browser: 'chromium',
          },
        ],
        provider: playwright(),
      },
      reporters: [
        [
          'json',
          {
            outputFile: 'vitest-browser-scenario-report.json',
            includeTaskLocation: true,
          },
        ],
      ],
    },
  })
  

  scenario.browser.test.ts

  import {expect, test} from 'vitest'
  
  test('browser test', () => {
    expect(1).toBe(1)
  })
  

• eslint

  Example for reporting on eslint checks

  eslint.config.scenario.js

  import {defineConfig} from 'eslint/config'
  import tseslint from 'typescript-eslint'
  
  export default defineConfig({
    files: ['src/**/*.{ts,tsx}'],
    extends: [tseslint.configs.recommended],
  })
  

Scenario configuration

import {defineConfig} from '@primer/agent-eval/scenario'
import type {ESLint} from 'eslint'
import type {JsonTestResults} from 'vitest/node'

export default defineConfig({
  prompt: `Example prompt`,
  checks: [
    {
      name: 'node-tests',
      description: 'Example for node-based tests with vitest',
      files: ['vitest.config.scenario.ts', 'scenario.test.ts'],
      async run({sandbox}) {
        const commandResult = await sandbox.runCommand(
          'npx',
          ['vitest', 'run', '--config', 'vitest.config.scenario.ts'],
          {
            allowNonZeroExitCode: true,
          },
        )

        if (commandResult.exitCode !== 0 && commandResult.exitCode !== 1) {
          throw new Error(`Vitest failed with exit code ${commandResult.exitCode}: ${commandResult.stderr}`)
        }

        const contents = await sandbox.readFile('vitest-scenario-report.json')
        const json: JsonTestResults = JSON.parse(contents)

        return {
          outcomes: json.testResults.flatMap(({assertionResults}) => {
            return assertionResults.map(assertionResult => {
              return {
                type: 'outcome',
                status:
                  assertionResult.status === 'passed'
                    ? 'passed'
                    : assertionResult.status === 'failed'
                      ? 'failed'
                      : 'skipped',
              }
            })
          }),
        }
      },
    },
    {
      name: 'browser-tests',
      description: 'Example for browser-based tests with vitest',
      files: ['vitest.config.browser.scenario.ts', 'scenario.browser.test.ts'],
      async run({sandbox}) {
        await sandbox.runCommand('npx', ['playwright', 'install'])

        const commandResult = await sandbox.runCommand(
          'npx',
          ['vitest', 'run', '--config', 'vitest.config.browser.scenario.ts'],
          {
            allowNonZeroExitCode: true,
          },
        )

        if (commandResult.exitCode !== 0 && commandResult.exitCode !== 1) {
          throw new Error(`Vitest failed with exit code ${commandResult.exitCode}: ${commandResult.stderr}`)
        }

        const contents = await sandbox.readFile('vitest-browser-scenario-report.json')
        const json: JsonTestResults = JSON.parse(contents)

        return {
          outcomes: json.testResults.flatMap(({assertionResults}) => {
            return assertionResults.map(assertionResult => {
              return {
                type: 'outcome',
                status:
                  assertionResult.status === 'passed'
                    ? 'passed'
                    : assertionResult.status === 'failed'
                      ? 'failed'
                      : 'skipped',
              }
            })
          }),
        }
      },
    },
    {
      name: 'eslint',
      description: 'Example for reporting on eslint checks',
      files: ['eslint.config.scenario.js'],
      async run({sandbox}) {
        const commandResult = await sandbox.runCommand(
          'npx',
          [
            'eslint',
            '--config',
            'eslint.config.scenario.js',
            'src',
            '--format',
            'json',
            '--output-file',
            'eslint-scenario-report.json',
            '--max-warnings',
            '0',
          ],
          {
            allowNonZeroExitCode: true,
          },
        )

        if (commandResult.exitCode !== 0 && commandResult.exitCode !== 1) {
          throw new Error(`ESLint failed with exit code ${commandResult.exitCode}: ${commandResult.stderr}`)
        }

        const contents = await sandbox.readFile('eslint-scenario-report.json')
        const results: Array<ESLint.LintResult> = JSON.parse(contents)

        return {
          outcomes: results.map(result => {
            return {
              type: 'outcome',
              id: result.filePath,
              status: result.errorCount === 0 && result.warningCount === 0 ? 'passed' : 'failed',
            }
          }),
        }
      },
    },
  ],
})

Experiments

This scenario is not included in any experiments.