Interface IEvaluator

Interface for the agent evaluator.

Example

const evaluator = new Evaluator();

// Create test suite
const testCases: EvalTestCase[] = [
  {
    id: 'greet-1',
    name: 'Basic greeting',
    input: 'Hello!',
    expectedOutput: 'Hello! How can I help you today?',
    criteria: [
      { name: 'relevance', description: 'Is greeting appropriate', weight: 0.5, scorer: 'llm_judge' },
      { name: 'politeness', description: 'Is response polite', weight: 0.5, scorer: 'contains' },
    ],
  },
];

// Run evaluation
const run = await evaluator.runEvaluation('greeting-test', testCases, agentFn);
console.log(`Pass rate: ${run.aggregateMetrics.passRate * 100}%`);
Copy