Evaluate agents with agent-eval

pnpm add @tangle-network/agent-eval @tangle-network/agent-runtime @tangle-network/sandbox

import { defineAgentEval } from "@tangle-network/agent-eval/contract";
 
interface SupportCase {
  id: string;
  kind: "support";
}
 
const evalKit = defineAgentEval<SupportCase, string>({
  scenarios: [
    { id: "refund", kind: "support" },
    { id: "shipping", kind: "support" },
    { id: "cancel", kind: "support" },
  ],
 
  // The agent under test: given a case, return a response.
  run: async (scenario) => {
    // Replace with a real agent call:
    // const result = await runAgentRounds({ task: scenario.id, ... });
    return `Handled ${scenario.id} request`;
  },
 
  // The judge: given a case and a response, return a verdict.
  judge: async (scenario, response) => {
    const passed = response.includes(scenario.id);
    return { passed, reason: passed ? "Correctly handled" : "Missing case reference" };
  },
});
 
// Score one candidate
const result = await evalKit.score({
  candidate: { id: "my-agent-v1", createdAt: new Date().toISOString() },
});
 
console.log(`Passed: ${result.passed}/${result.total} scenarios`);
console.log(`Cost: $${result.totalCostUsd.toFixed(4)}`);

import { compareEvals } from "@tangle-network/agent-eval/comparison";
 
// Run both candidates
const baseline = await evalKit.score({
  candidate: { id: "agent-v1", createdAt: new Date().toISOString() },
});
 
const candidate = await evalKit.score({
  candidate: { id: "agent-v2", createdAt: new Date().toISOString() },
});
 
// Compare them
const comparison = await compareEvals({
  baseline,
  candidate,
  config: {
    // Minimum improvement to declare a win
    minImprovement: 0.05,
    // Confidence level for the statistical test
    confidenceLevel: 0.95,
  },
});
 
console.log(comparison.verdict);  // "improved" | "regressed" | "inconclusive"
console.log(comparison.evidence);  // Full evidence record

import { Sandbox } from "@tangle-network/sandbox";
import { createTangleProvider } from "@tangle-network/agent-provider-tangle";
import { providerAsSandboxClient, runAgentRounds } from "@tangle-network/agent-runtime/kernel";
import { defineAgentEval } from "@tangle-network/agent-eval/contract";
 
const sandbox = new Sandbox({
  apiKey: process.env.TANGLE_API_KEY!,
  baseUrl: "https://sandbox.tangle.tools",
});
const provider = createTangleProvider({ client: sandbox });
const client = providerAsSandboxClient(provider);
 
const evalKit = defineAgentEval<SupportCase, string>({
  scenarios: [
    { id: "refund", kind: "support" },
    { id: "shipping", kind: "support" },
    { id: "cancel", kind: "support" },
    { id: "upgrade", kind: "support" },
    { id: "downgrade", kind: "support" },
  ],
 
  run: async (scenario) => {
    const result = await runAgentRounds({
      task: `A customer asks about ${scenario.id}. Write a helpful response.`,
      driver: {
        plan: async (task, history) => (history.length === 0 ? [task] : []),
        decide: () => "done" as const,
      },
      agentRun: {
        profile: {
          name: "support-agent",
          harness: "claude-code",
          model: { provider: "anthropic", default: "anthropic/claude-sonnet-4-6" },
        },
        taskToPrompt: (t) => t,
      },
      output: { parse: (events) => JSON.stringify(events) },
      ctx: { sandboxClient: client },
    });
    return result.iterations[0]?.output ?? "";
  },
 
  judge: async (scenario, response) => {
    // Use a model to judge, or write a deterministic check
    const addressesTopic = response.toLowerCase().includes(scenario.id);
    const isHelpful = response.length > 20;
    return {
      passed: addressesTopic && isHelpful,
      reason: addressesTopic ? (isHelpful ? "Good" : "Too short") : "Off topic",
    };
  },
});
 
const result = await evalKit.score({
  candidate: { id: "support-agent-v2", createdAt: new Date().toISOString() },
});

import { createImprovementGate } from "@tangle-network/agent-eval/improvement";
 
const gate = createImprovementGate({
  // Split scenarios into dev (optimize on) and final (prove on)
  split: { devFraction: 0.7, seed: "fixed-seed" },
 
  // The optimizer explores on the dev partition only
  optimize: async (devScenarios) => {
    // Try different prompts, models, or tool configurations
    // Return the best candidate found on dev
    return { profile: optimizedProfile };
  },
 
  // The gate proves on final — the optimizer never sees these
  prove: async (finalScenarios, candidate) => {
    return evalKit.score({ candidate });
  },
});
 
const decision = await gate.run();
// decision: "promoted" | "rejected" | "inconclusive"

import { createModelTransport } from "@tangle-network/agent-eval/transport";
 
const transport = createModelTransport({
  // Route through Tangle Router
  provider: "tangle",
  apiKey: process.env.TANGLE_API_KEY!,
  baseUrl: "https://router.tangle.tools/v1",
 
  // Pricing: how much each model costs per token
  pricing: {
    "anthropic/claude-sonnet-4-6": {
      input: 0.000003,
      output: 0.000015,
    },
    "groq/llama-3.1-8b-instant": {
      input: 0.00000005,
      output: 0.00000008,
    },
  },
});