feat(eval): rename --evaluator to --evaluators

src/handlers/eval/ab-test/ab-test.fixture.test.tsx

@@ -199,7 +199,7 @@ describe("eval ab-test config-based run", () => {
       ONLINE_EVAL_NAME,
       "--agent",
       AGENT_ID,
-      "--evaluator",
+      "--evaluators",
       EVALUATOR_ID,
       "--sampling-rate",
       "100",
@@ -280,7 +280,7 @@ describe("eval ab-test target-based run", () => {
       TB_ONLINE_EVAL_C,
       "--agent",
       AGENT_ID,
-      "--evaluator",
+      "--evaluators",
       EVALUATOR_ID,
       "--sampling-rate",
       "100",
@@ -300,7 +300,7 @@ describe("eval ab-test target-based run", () => {
       TB_ONLINE_EVAL_T1,
       "--agent",
       AGENT_ID,
-      "--evaluator",
+      "--evaluators",
       EVALUATOR_ID,
       "--sampling-rate",
       "100",

src/handlers/eval/batch-evaluation/batch-evaluation.fixture.test.tsx

@@ -128,7 +128,7 @@ describe("eval batch-evaluation (fixture-backed)", () => {
       "evaluate",
       "--agent",
       FIXTURE_EVAL_AGENT,
-      "--evaluator",
+      "--evaluators",
       "Builtin.Helpfulness",
       "--name",
       FIXTURE_EVAL_NAME,
@@ -172,7 +172,7 @@ describe("eval batch-evaluation (fixture-backed)", () => {
       '{"prompt":"{input}"}',
       "--dataset",
       FIXTURE_SIMULATE_DATASET,
-      "--evaluator",
+      "--evaluators",
       "Builtin.Helpfulness",
       "--name",
       FIXTURE_SIMULATE_NAME,

src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx

@@ -172,7 +172,7 @@ describe("eval batch-evaluation simulate", () => {
     '{"prompt":"{input}"}',
     "--dataset",
     "/tmp/ds.jsonl",
-    "--evaluator",
+    "--evaluators",
     "Builtin.Helpfulness",
     "--name",
     "sim-1",
@@ -181,18 +181,27 @@ describe("eval batch-evaluation simulate", () => {
   test.each<[RegExp, string[]]>([
     [
       /--runtime-id/,
-      ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"],
+      [
+        "--payload-template",
+        "{}",
+        "--dataset",
+        "/tmp/ds.jsonl",
+        "--evaluators",
+        "E",
+        "--name",
+        "n",
+      ],
     ],
     [
       /--payload-template/,
-      ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"],
+      ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluators", "E", "--name", "n"],
     ],
     [
       /--dataset/,
-      ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E", "--name", "n"],
+      ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluators", "E", "--name", "n"],
     ],
     [
-      /--evaluator/,
+      /--evaluators/,
       [
         "--runtime-id",
         "r-1",
@@ -213,7 +222,7 @@ describe("eval batch-evaluation simulate", () => {
         "{}",
         "--dataset",
         "/tmp/ds.jsonl",
-        "--evaluator",
+        "--evaluators",
         "E",
       ],
     ],

src/handlers/eval/batch-evaluation/evaluate/index.tsx

@@ -14,7 +14,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
     description: "evaluate existing sessions service-side (async; returns a job ID)",
     flags: [
       ...SessionSource.flags,
-      flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
       flag(
         "ground-truth",
         "session ground truth (JSON SessionMetadataShape[]; inline, file://<path>, or -)",
@@ -28,9 +28,9 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
       if (!flags["name"]) {
         throw new InputValidationError("required option '--name <name>' not specified");
       }
-      if (!flags["evaluator"] || flags["evaluator"].length === 0) {
+      if (!flags["evaluators"] || flags["evaluators"].length === 0) {
         throw new InputValidationError(
-          "required option '--evaluator <evaluator...>' not specified",
+          "required option '--evaluators <evaluators...>' not specified",
         );
       }
 
@@ -46,7 +46,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
         {
           name: flags["name"],
           description: flags["description"],
-          evaluatorIds: flags["evaluator"],
+          evaluatorIds: flags["evaluators"],
           source,
           groundTruth,
           kmsKeyArn: flags["kms-key-arn"],

src/handlers/eval/batch-evaluation/simulate/index.tsx

@@ -33,7 +33,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) =>
       flag("user-id", "Runtime user ID", z.string().optional()),
       flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()),
       flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()),
-      flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
       flag("name", "batch evaluation name (unique in the account)", z.string().optional()),
       flag("description", "description for the batch evaluation", z.string().optional()),
       flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()),
@@ -51,9 +51,9 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) =>
       }
       if (!flags["dataset"])
         throw new InputValidationError("required option '--dataset' not specified");
-      if (!flags["evaluator"]?.length) {
+      if (!flags["evaluators"]?.length) {
         throw new InputValidationError(
-          "required option '--evaluator <evaluator...>' not specified",
+          "required option '--evaluators <evaluators...>' not specified",
         );
       }
       if (!flags["name"])
@@ -95,7 +95,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) =>
           {
             name: flags["name"],
             description: flags["description"],
-            evaluatorIds: flags["evaluator"],
+            evaluatorIds: flags["evaluators"],
             source: {
               origin: "agent",
               agent: flags["runtime-id"],

src/handlers/eval/batch-insights/batch-insights.test.tsx

@@ -137,7 +137,7 @@ describe("eval batch-insights run", () => {
       "--insight",
       "Builtin.Insight.UserIntent",
       "Builtin.Insight.ExecutionSummary",
-      "--evaluator",
+      "--evaluators",
       "Builtin.Helpfulness",
       "--json",
     ]);

src/handlers/eval/batch-insights/run/index.tsx

@@ -17,7 +17,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) =>
       ...SessionSource.flags,
       flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT])),
       flag(
-        "evaluator",
+        "evaluators",
         "optional evaluator ID(s) to run alongside the insights",
         z.array(z.string()).optional(),
       ),
@@ -36,7 +36,7 @@ export const createRunBatchInsightsHandler = (core: Core, io: AppIO) =>
           name: flags["name"],
           description: flags["description"],
           insightIds: flags["insight"],
-          evaluatorIds: flags["evaluator"],
+          evaluatorIds: flags["evaluators"],
           source,
           kmsKeyArn: flags["kms-key-arn"],
         },

src/handlers/eval/ondemand/evaluate/index.tsx

@@ -19,7 +19,7 @@ export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) =>
         z.string().optional(),
       ),
       flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()),
-      flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
       flag(
         "lookback-days",
         "time filter: evaluate sessions from the last N days",
@@ -51,9 +51,9 @@ export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) =>
       if (!flags["agent"]) {
         throw new InputValidationError("on-demand requires '--agent'");
       }
-      if (!flags["evaluator"] || flags["evaluator"].length === 0) {
+      if (!flags["evaluators"] || flags["evaluators"].length === 0) {
         throw new InputValidationError(
-          "required option '--evaluator <evaluator...>' not specified",
+          "required option '--evaluators <evaluators...>' not specified",
         );
       }
 
@@ -84,7 +84,7 @@ export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) =>
       );
 
       const result = await core.eval.evaluate(
-        { traces, evaluatorIds: flags["evaluator"], groundTruth },
+        { traces, evaluatorIds: flags["evaluators"], groundTruth },
         opts,
       );
       ctx.require(JsonRendererKey).renderJson(result);

src/handlers/eval/ondemand/ondemand.fixture.test.tsx

@@ -61,7 +61,7 @@ describe("eval ondemand evaluate (fixture-backed)", () => {
       WINDOW_START,
       "--end-time",
       WINDOW_END,
-      "--evaluator",
+      "--evaluators",
       "Builtin.Helpfulness",
     ]);
 
@@ -113,7 +113,7 @@ describe("eval ondemand evaluate (fixture-backed)", () => {
       '{"prompt":"{input}"}',
       "--dataset",
       SIMULATE_DATASET,
-      "--evaluator",
+      "--evaluators",
       "Builtin.Correctness",
       "--ingestion-wait-ms",
       isRecording() ? "150000" : "0",

src/handlers/eval/ondemand/ondemand.test.tsx

@@ -141,7 +141,7 @@ async function runWithRealCore(options: LogsOptions, logger = createSilentLogger
     "evaluate",
     "--agent",
     RUNTIME_ID,
-    "--evaluator",
+    "--evaluators",
     "Builtin.Helpfulness",
     "--session-ids",
     "session-1",
@@ -163,7 +163,7 @@ const BASE = [
   "evaluate",
   "--agent",
   "a-1",
-  "--evaluator",
+  "--evaluators",
   "Builtin.Helpfulness",
 ];
 
@@ -195,22 +195,22 @@ describe("eval ondemand simulate", () => {
     '{"prompt":"{input}"}',
     "--dataset",
     "/tmp/ds.jsonl",
-    "--evaluator",
+    "--evaluators",
     "Builtin.Helpfulness",
   ];
 
   test.each<[RegExp, string[]]>([
     [
       /--runtime-id/,
-      ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"],
+      ["--payload-template", "{}", "--dataset", "/tmp/ds.jsonl", "--evaluators", "E"],
     ],
     [
       /--payload-template/,
-      ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E"],
+      ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluators", "E"],
     ],
-    [/--dataset/, ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E"]],
+    [/--dataset/, ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluators", "E"]],
     [
-      /--evaluator/,
+      /--evaluators/,
       ["--runtime-id", "r-1", "--payload-template", "{}", "--dataset", "/tmp/ds.jsonl"],
     ],
   ])("rejects when a required flag is missing (%s)", async (expected, args) => {
@@ -280,13 +280,21 @@ describe("eval ondemand evaluate validation", () => {
   test.each<[string, string[], RegExp]>([
     [
       "requires --agent",
-      ["eval", "ondemand", "evaluate", "--evaluator", "Builtin.Helpfulness", "--session-ids", "s1"],
+      [
+        "eval",
+        "ondemand",
+        "evaluate",
+        "--evaluators",
+        "Builtin.Helpfulness",
+        "--session-ids",
+        "s1",
+      ],
       /--agent/,
     ],
     [
-      "requires --evaluator",
+      "requires --evaluators",
       ["eval", "ondemand", "evaluate", "--agent", "a-1", "--session-ids", "s1"],
-      /--evaluator/,
+      /--evaluators/,
     ],
     ["rejects an empty session source", BASE, /session source/],
     [

src/handlers/eval/ondemand/simulate/index.tsx

@@ -34,7 +34,7 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) =>
       flag("user-id", "Runtime user ID", z.string().optional()),
       flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()),
       flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()),
-      flag("evaluator", "evaluator ID(s) to apply", z.array(z.string()).optional()),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
       flag(
         "ingestion-wait-ms",
         "ms to wait for span ingestion before grading (default 180000; 0 to skip)",
@@ -49,16 +49,16 @@ export const createSimulateOnDemandHandler = (core: Core, _io: AppIO) =>
       }
       if (!flags["dataset"])
         throw new InputValidationError("required option '--dataset' not specified");
-      if (!flags["evaluator"]?.length) {
+      if (!flags["evaluators"]?.length) {
         throw new InputValidationError(
-          "required option '--evaluator <evaluator...>' not specified",
+          "required option '--evaluators <evaluators...>' not specified",
         );
       }
 
       const runtimeId = flags["runtime-id"];
       const payloadTemplate = flags["payload-template"];
       const dataset = flags["dataset"];
-      const evaluatorIds = flags["evaluator"];
+      const evaluatorIds = flags["evaluators"];
 
       await withUserCancellation(async (signal) => {
         const opts = coreOptsFromCtx(ctx);

src/handlers/eval/online-eval/create/index.tsx

@@ -24,7 +24,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) =>
         "the traces to evaluate (JSON DataSourceConfig; inline, file://<path>, or - for stdin), as an alternative to --agent",
         z.string().optional(),
       ),
-      flag("evaluator", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()),
+      flag("evaluators", "the ID(s) of the evaluators to apply", z.array(z.string()).optional()),
       flag(
         "sampling-rate",
         "percentage of sessions to sample (0.01-100)",
@@ -64,9 +64,9 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) =>
           "required option '--sampling-rate <sampling-rate>' not specified",
         );
       }
-      if (!flags["evaluator"] || flags["evaluator"].length === 0) {
+      if (!flags["evaluators"] || flags["evaluators"].length === 0) {
         throw new InputValidationError(
-          "required option '--evaluator <evaluator...>' not specified",
+          "required option '--evaluators <evaluators...>' not specified",
         );
       }
 
@@ -91,7 +91,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) =>
           "filters",
           await source.resolveText("filters", flags["filters"]),
         ),
-        evaluatorIds: flags["evaluator"],
+        evaluatorIds: flags["evaluators"],
         evaluationExecutionRoleArn: flags["role-arn"],
         enableOnCreate:
           flags["enable-on-create"] === undefined