From 50a675355e715fbc6deb0b0689df58f246d08812 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Thu, 10 Sep 2026 23:48:42 +0000 Subject: [PATCH] feat(online-eval): add --output-config and --tags, --description on update --output-config lets a customer say where results and metrics are written. Same passthrough contract as batch evaluation's flag of the same name, but a separate module and a separate generated type: the online-evaluation OutputConfig is a plain object with no logStreamName, while the batch one is a tagged union that has one. --tags on create reuses TagsSchema and parseJsonFlagWithSchema, matching `project add memory`, so a non-string tag value is rejected here rather than by the API. --description on update was simply missing. The execution role is what makes the destination usable. Its policy granted result writes only to /aws/bedrock-agentcore/evaluations/*, so a CLI-managed role plus a custom log group would have produced a config whose results could not be written anywhere. executionPolicy now also grants the customer-named group, or the sampled source groups for SOURCE_LOG_GROUP, and an output change joins a data-source change as a trigger for re-scoping. A single ARN stays a bare string rather than a one-element array. IAM treats those identically, but the document's exact text is hashed to name the inline policy, so wrapping it would rename the policy attached to every config that has no custom destination. A destination already inside the reserved /aws/bedrock-agentcore/evaluations/ namespace adds nothing, keeping the document stable for a config whose stored outputConfig is the service-managed default the API echoes back. RoleScopeWarning carries which half of the scope moved, because the permissions the caller has to add differ: querying traces needs logs:StartQuery and logs:GetQueryResults on the sampled groups, while writing results needs logs:PutLogEvents and, for a group that does not exist yet, logs:CreateLogGroup. --- src/core/eval.tsx | 53 +++++- src/core/onlineEvalExecutionRole.test.ts | 65 +++++++ src/core/onlineEvalExecutionRole.tsx | 41 +++- .../eval/online-eval/create/index.tsx | 30 ++- .../online-eval/online-eval.flags.test.tsx | 178 ++++++++++++++++++ .../eval/online-eval/outputConfig.tsx | 58 ++++++ .../eval/online-eval/update/index.tsx | 34 +++- src/handlers/eval/types.tsx | 15 +- src/testing/TestCoreClient.tsx | 17 +- 9 files changed, 467 insertions(+), 24 deletions(-) create mode 100644 src/handlers/eval/online-eval/online-eval.flags.test.tsx create mode 100644 src/handlers/eval/online-eval/outputConfig.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 61bd76802..28b417c3c 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -123,6 +123,8 @@ import type { DatasetUpdateProgressEvent, DatasetUpdateResult, RoleScopeWarning, + RoleScopeKind, + OnlineEvalOutputConfig, CoreEvalClient, CreateConfigurationBundleInput, CreateConfigBasedABTestInput, @@ -989,6 +991,9 @@ export class EvalClient implements CoreEvalClient { options.region, logGroupNamesOf(dataSourceConfig), await evaluatorKmsKeys(input.evaluatorIds ?? [], control), + // Read only to widen the write scope to the chosen destination; the + // request object below still gets the caller's object untouched. + { outputConfig: input.outputConfig }, ) ).roleArn; @@ -998,8 +1003,10 @@ export class EvalClient implements CoreEvalClient { rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), dataSourceConfig, evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + outputConfig: input.outputConfig, evaluationExecutionRoleArn, enableOnCreate: input.enableOnCreate ?? true, + tags: input.tags, }); // A role provisioned moments ago may not be assumable yet (IAM is eventually @@ -1237,6 +1244,11 @@ export class EvalClient implements CoreEvalClient { ? dataSourceConfig : undefined; + const outputMoved = update.outputConfig !== undefined; + const effectiveOutputConfig = update.outputConfig ?? current.outputConfig; + const scopeKind: RoleScopeKind = + movedTo !== undefined && outputMoved ? "input-and-output" : outputMoved ? "output" : "input"; + const configName = current.onlineEvaluationConfigName; const roleArn = update.evaluationExecutionRoleArn ?? current.evaluationExecutionRoleArn; const managedRoleName = @@ -1246,26 +1258,33 @@ export class EvalClient implements CoreEvalClient { isManagedOnlineEvalRole(roleArn, configName) ? configName : undefined; - const refreshManagedRole = movedTo !== undefined && managedRoleName !== undefined; - - if (movedTo !== undefined && managedRoleName === undefined && roleArn) { + const scopeChanged = movedTo !== undefined || outputMoved; + const refreshManagedRole = scopeChanged && managedRoleName !== undefined; + const affectedLogGroups = [ + ...(movedTo !== undefined ? logGroupNamesOf(movedTo) : []), + ...(outputMoved ? destinationLogGroupNames(update.outputConfig, dataSourceConfig) : []), + ]; + + if (scopeChanged && managedRoleName === undefined && roleArn) { roleScopeWarning = { reason: "custom-role", roleArn, - logGroupNames: logGroupNamesOf(movedTo), + scope: scopeKind, + logGroupNames: affectedLogGroups, }; - } else if (movedTo !== undefined && !refreshManagedRole && roleArn) { + } else if (scopeChanged && !refreshManagedRole && roleArn) { // managed role, but the caller declined the refresh roleScopeWarning = { reason: "update-declined", roleArn, - logGroupNames: logGroupNamesOf(movedTo), + scope: scopeKind, + logGroupNames: affectedLogGroups, }; } if (refreshManagedRole && update.updateRole !== false) { const iam = this.clients.iam({ region: options.region }); - const newLogGroups = logGroupNamesOf(movedTo); + const newLogGroups = dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : []; const oldLogGroups = current.dataSourceConfig ? logGroupNamesOf(current.dataSourceConfig) : []; @@ -1290,7 +1309,7 @@ export class EvalClient implements CoreEvalClient { options.region, newLogGroups, kmsKeys, - resourceNameFromArn(roleArn!), + { roleName: resourceNameFromArn(roleArn!), outputConfig: effectiveOutputConfig }, ); const oldPolicyName = scopePolicyName( executionPolicy( @@ -1298,15 +1317,18 @@ export class EvalClient implements CoreEvalClient { accountIdFromRoleArn(managedRoleArn), oldLogGroups, kmsKeys, + current.outputConfig, ), ); const response = await control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, + description: update.description, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), dataSourceConfig, evaluators, + outputConfig: update.outputConfig, }), ); @@ -1323,6 +1345,7 @@ export class EvalClient implements CoreEvalClient { roleScopeWarning = { reason: "stale-scope", roleArn: roleArn!, + scope: scopeKind, logGroupNames: oldLogGroups, }; } @@ -1333,9 +1356,11 @@ export class EvalClient implements CoreEvalClient { const response = await control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, + description: update.description, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), dataSourceConfig, evaluators, + outputConfig: update.outputConfig, evaluationExecutionRoleArn: update.evaluationExecutionRoleArn, }), ); @@ -2201,6 +2226,18 @@ function logGroupNamesOf(dataSourceConfig: DataSourceConfig): string[] { : []; } +function destinationLogGroupNames( + outputConfig: OnlineEvalOutputConfig | undefined, + dataSourceConfig: DataSourceConfig | undefined, +): string[] { + const cloudWatch = outputConfig?.cloudWatchConfig; + if (!cloudWatch) return []; + if (cloudWatch.resultDestination === "SOURCE_LOG_GROUP") { + return dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : []; + } + return cloudWatch.logGroupName ? [cloudWatch.logGroupName] : []; +} + // runtimeIdFromLogGroup recovers the runtime id embedded in a log group path // produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for a // new --endpoint without the caller passing --agent again. Returns undefined for diff --git a/src/core/onlineEvalExecutionRole.test.ts b/src/core/onlineEvalExecutionRole.test.ts index 7f1ec01ea..e4a719410 100644 --- a/src/core/onlineEvalExecutionRole.test.ts +++ b/src/core/onlineEvalExecutionRole.test.ts @@ -110,3 +110,68 @@ test("gives identical policies the same name", () => { scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])), ); }); + +function writeStatement(policy: string) { + return statements(policy).find((s) => s.Sid === "WriteEvaluationResults"); +} + +const SERVICE_RESULTS = `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/evaluations/*`; + +test("a config with no output destination keeps the service namespace as a bare string", () => { + const write = writeStatement(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])); + + expect(write?.Resource).toBe(SERVICE_RESULTS); + expect(Array.isArray(write?.Resource)).toBe(false); +}); + +test("a customer-named dedicated group is granted alongside the service namespace", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/company/agent-evaluations*`, + ]); + expect(write?.Action).toContain("logs:CreateLogGroup"); +}); + +test("SOURCE_LOG_GROUP grants writes to the groups the traces are read from", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`, + ]); +}); + +test("a destination already inside the service namespace adds nothing", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/aws/bedrock-agentcore/evaluations/online-evaluations/results/default", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toBe(SERVICE_RESULTS); +}); + +test("changing the destination changes the policy name, so a re-scope is a new grant", () => { + const before = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []); + const after = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { logGroupName: "/company/agent-evaluations" }, + }); + + expect(scopePolicyName(after)).not.toBe(scopePolicyName(before)); +}); diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx index e02a6f40a..4c1896b6e 100644 --- a/src/core/onlineEvalExecutionRole.tsx +++ b/src/core/onlineEvalExecutionRole.tsx @@ -15,7 +15,6 @@ import { parseArn, resourceNameFromArn } from "./arn"; // evaluation results back to CloudWatch. When the caller doesn't bring one, // OnlineEvalClient provisions a per-config default here, scoped to the log // group(s) being sampled. Idempotent: an existing role is reused. -// // Each scope is stored as its own inline policy, named after a fingerprint of the // scope, so granting a new scope never overwrites the policy backing the current // one. IAM unions Allows across a role's inline policies, which lets an update @@ -83,10 +82,35 @@ function runtimeLogGroupPrefix(logGroupName: string): string { return match?.[1] ?? logGroupName; } +const SERVICE_RESULT_PREFIX = "/aws/bedrock-agentcore/evaluations/"; + +function resultWriteArns( + logs: string, + sampledArns: string[], + outputConfig: OnlineEvalResultDestination | undefined, +): string | string[] { + const arns = [`${logs}:${SERVICE_RESULT_PREFIX}*`]; + const cloudWatch = outputConfig?.cloudWatchConfig; + + if (cloudWatch?.resultDestination === "SOURCE_LOG_GROUP") { + arns.push(...sampledArns); + } else if ( + cloudWatch?.logGroupName && + !cloudWatch.logGroupName.startsWith(SERVICE_RESULT_PREFIX) + ) { + arns.push(`${logs}:${cloudWatch.logGroupName}*`); + } + + return arns.length === 1 ? arns[0]! : arns; +} + +export type OnlineEvalResultDestination = { + cloudWatchConfig?: { logGroupName?: string; resultDestination?: string } | undefined; +}; + // executionPolicy grants the permissions CreateOnlineEvaluationConfig validates // at creation time. Exported for assertion: the policy body is not observable // through the recorded IAM fixtures, whose responses are empty. -// // at creation time: Logs Insights query access over the sampled log groups plus // the `aws/spans` group that carries the actual trace spans, Bedrock model // invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based @@ -98,6 +122,7 @@ export function executionPolicy( accountId: string, logGroupNames: string[], kmsKeyArns: string[], + outputConfig?: OnlineEvalResultDestination, ): string { const logs = `arn:aws:logs:${region}:${accountId}:log-group`; const spansArn = `${logs}:aws/spans`; @@ -134,6 +159,8 @@ export function executionPolicy( Resource: [`${spansArn}*`, ...sampledArns], }, { + // logs:CreateLogGroup is needed because the service creates a + // customer-named result group that does not exist yet. Sid: "WriteEvaluationResults", Effect: "Allow", Action: [ @@ -142,7 +169,7 @@ export function executionPolicy( "logs:DescribeLogStreams", "logs:PutLogEvents", ], - Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`, + Resource: resultWriteArns(logs, sampledArns, outputConfig), }, { Sid: "IndexSpans", @@ -201,6 +228,11 @@ export function scopePolicyName(policyDocument: string): string { return `${POLICY_PREFIX}-${fingerprint(policyDocument)}`; } +export type GrantScopeOptions = { + roleName?: string; + outputConfig?: OnlineEvalResultDestination; +}; + // grantOnlineEvalScope creates the execution role for `configName` if it does not // exist and attaches the inline policy for this scope, returning the role ARN and // the policy name written. The caller revokes the superseded scope once whatever @@ -211,7 +243,7 @@ export async function grantOnlineEvalScope( region: string, logGroupNames: string[], kmsKeyArns: string[] = [], - roleName = onlineEvalExecutionRoleName(configName), + { roleName = onlineEvalExecutionRoleName(configName), outputConfig }: GrantScopeOptions = {}, ): Promise<{ roleArn: string; policyName: string }> { let roleArn: string; try { @@ -234,6 +266,7 @@ export async function grantOnlineEvalScope( accountIdFromRoleArn(roleArn), logGroupNames, kmsKeyArns, + outputConfig, ); const policyName = scopePolicyName(policyDocument); await iam.send( diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index 273b95c7b..c28b7bb49 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -5,9 +5,24 @@ import { InputValidationError } from "../../../../errors"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; -import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { + assertMutuallyExclusiveFlags, + coreOptsFromCtx, + parseJsonFlag, + parseJsonFlagWithSchema, +} from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; +import { TagsSchema } from "../../../../projectSchemas/tags"; + +const tagsHelp = `(JSON: map of string to string) +Tags applied to the online evaluation configuration. + +Accepts inline JSON, file://, or - to read stdin. + +Example: + --tags '{"team":"ml-platform","env":"prod"}'`; const CONFIGURATION = "Configuration:"; const SESSION_SOURCE = "Session source (choose exactly one):"; @@ -35,6 +50,10 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => z.enum(["true", "false"]).optional(), { group: CONFIGURATION }, ), + flag("tags", "resource tags (JSON object of key/value strings)", z.string().optional(), { + group: CONFIGURATION, + help: tagsHelp, + }), flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { group: SESSION_SOURCE, }), @@ -69,6 +88,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", @@ -98,9 +118,17 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => } const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], io); + const tags = parseJsonFlagWithSchema( + "tags", + await source.resolveText("tags", flags["tags"]), + TagsSchema, + ); const common = { name: flags["name"], description: flags["description"], + tags, + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( diff --git a/src/handlers/eval/online-eval/online-eval.flags.test.tsx b/src/handlers/eval/online-eval/online-eval.flags.test.tsx new file mode 100644 index 000000000..e198386ce --- /dev/null +++ b/src/handlers/eval/online-eval/online-eval.flags.test.tsx @@ -0,0 +1,178 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; +import type { CreateOnlineEvalInput, UpdateOnlineEvalInput } from "../types"; + +async function run( + args: string[], + configure?: (core: TestCoreClient) => void, + ioOptions?: { stdin?: string }, +) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(ioOptions); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +function createInput(core: TestCoreClient): CreateOnlineEvalInput { + const call = core.eval.calls.find((c) => c.method === "createOnlineEvaluationConfig"); + expect(call).toBeDefined(); + return call!.args[0] as CreateOnlineEvalInput; +} + +function updateInput(core: TestCoreClient): UpdateOnlineEvalInput { + const call = core.eval.calls.find((c) => c.method === "updateOnlineEvaluationConfig"); + expect(call).toBeDefined(); + return call!.args[1] as UpdateOnlineEvalInput; +} + +const OUTPUT_CONFIG = { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + metricsNamespace: "Company/AgentEvaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, +} as const; + +describe("eval online-eval create", () => { + const BASE = [ + "eval", + "online-eval", + "create", + "--name", + "quality", + "--agent", + "r-1", + "--evaluators", + "Builtin.Helpfulness", + "--sampling-rate", + "10", + ]; + + test("--output-config reaches Core unchanged from inline JSON", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(OUTPUT_CONFIG)]); + expect(createInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("--output-config reaches Core unchanged from stdin", async () => { + const { core } = await run([...BASE, "--output-config", "-"], undefined, { + stdin: JSON.stringify(OUTPUT_CONFIG), + }); + expect(createInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("--tags reaches Core as a parsed map", async () => { + const { core } = await run([...BASE, "--tags", '{"team":"ml-platform","env":"dev"}']); + expect(createInput(core).tags).toEqual({ team: "ml-platform", env: "dev" }); + }); + + test("both are left undefined when omitted, so the service keeps its defaults", async () => { + const { core } = await run(BASE); + expect(createInput(core).outputConfig).toBeUndefined(); + expect(createInput(core).tags).toBeUndefined(); + }); + + test.each([ + ["--output-config", "{not json", /Invalid JSON for option '--output-config'/], + ["--tags", "{not json", /Invalid JSON for option '--tags'/], + ])("malformed %s fails before Core provisions anything", async (flag, value, expected) => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route(["node", "agentcore", ...BASE, flag, value, "--region", "us-west-2"]), + ).rejects.toThrow(expected); + expect(core.eval.calls).toEqual([]); + }); + + test("--tags rejects a non-string value rather than passing it to the API", async () => { + await expect(run([...BASE, "--tags", '{"team":42}'])).rejects.toThrow( + /Invalid value for option '--tags'/, + ); + }); +}); + +describe("eval online-eval update", () => { + const BASE = ["eval", "online-eval", "update", "--id", "online-eval-123"]; + + test("--description reaches Core", async () => { + const { core } = await run([...BASE, "--description", "checks tone on prod traffic"]); + expect(updateInput(core).description).toBe("checks tone on prod traffic"); + }); + + test("--output-config reaches Core unchanged", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(OUTPUT_CONFIG)]); + expect(updateInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("an omitted --output-config is not sent, so the destination is preserved", async () => { + const { core } = await run([...BASE, "--sampling-rate", "20"]); + expect(updateInput(core).outputConfig).toBeUndefined(); + }); + + test("malformed --output-config fails before the initial Get", async () => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route([ + "node", + "agentcore", + ...BASE, + "--output-config", + "{not json", + "--region", + "us-west-2", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + expect(core.eval.calls).toEqual([]); + }); + + test.each([ + ["input", "data source moved", "logs:StartQuery and logs:GetQueryResults"], + ["output", "output destination moved", "logs:PutLogEvents"], + ["input-and-output", "data source and output destination moved", "logs:PutLogEvents"], + ] as const)( + "a %s scope warning names what moved and what to grant", + async (scope, movedText, action) => { + const { stderr } = await run(BASE, (c) => + c.eval.setOnlineEvalRoleScopeWarning({ + reason: "custom-role", + roleArn: "arn:aws:iam::123456789012:role/MyRole", + scope, + logGroupNames: ["/company/agent-evaluations"], + }), + ); + expect(stderr).toContain(movedText); + expect(stderr).toContain(action); + expect(stderr).toContain("/company/agent-evaluations"); + }, + ); + + test("the scope warning is suppressed under --json", async () => { + const { stderr } = await run([...BASE, "--json"], (c) => + c.eval.setOnlineEvalRoleScopeWarning({ + reason: "custom-role", + roleArn: "arn:aws:iam::123456789012:role/MyRole", + scope: "output", + logGroupNames: ["/company/agent-evaluations"], + }), + ); + expect(stderr).toBe(""); + }); +}); diff --git a/src/handlers/eval/online-eval/outputConfig.tsx b/src/handlers/eval/online-eval/outputConfig.tsx new file mode 100644 index 000000000..b958592ae --- /dev/null +++ b/src/handlers/eval/online-eval/outputConfig.tsx @@ -0,0 +1,58 @@ +import z from "zod"; +import { SourceResolver, type AppIO } from "../../../io"; +import { flag } from "../../../router"; +import { parseJsonFlag } from "../../utils"; +import type { OnlineEvalOutputConfig } from "../types"; + +const outputConfigHelp = `(JSON object) +Where evaluation results and metrics are written. Omit it and results go to the +service-managed default location. Only top-level key: cloudWatchConfig. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchConfig": { + "logGroupName": "string", // result log group; omit for + // SOURCE_LOG_GROUP, and it cannot sit + // under /aws/bedrock-agentcore/evaluations/ + "metricsNamespace": "string", // CloudWatch metrics namespace + "resultDestination": "DEDICATED_LOG_GROUP" | "SOURCE_LOG_GROUP" + // DEDICATED_LOG_GROUP writes to a + // dedicated result group, creating it if + // needed; SOURCE_LOG_GROUP writes back to + // the groups the traces were sampled from + } + } + +A role you supply with --role-arn is never edited by the CLI, so it must already +grant logs:PutLogEvents on the destination — plus logs:CreateLogGroup when the +group does not exist yet. + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore-control/latest/APIReference/API_OutputConfig.html + +Example: + --output-config '{"cloudWatchConfig":{"logGroupName":"/company/agent-evaluations","metricsNamespace":"Company/AgentEvaluations","resultDestination":"DEDICATED_LOG_GROUP"}}'`; + +export class OnlineEvalOutputConfigFlag { + static readonly flags = [ + flag( + "output-config", + "where results and metrics are written (JSON OutputConfig)", + z.string().optional(), + { group: "Result output:", help: outputConfigHelp }, + ), + ] as const; + + static async resolve( + value: string | undefined, + io: AppIO, + ): Promise { + const resolver = new SourceResolver({ stdin: io.stdin }); + return parseJsonFlag( + "output-config", + await resolver.resolveText("output-config", value), + ); + } +} diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 6bdaa6807..57d932e30 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -6,15 +6,31 @@ import { JsonKey } from "../../../keys"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; +import type { RoleScopeKind } from "../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; const SESSION_SOURCE = "Session source:"; const SOURCE_FILTERS = "Source filters:"; const EVALUATION = "Evaluation:"; const EXECUTION = "Execution:"; +const MOVED: Record = { + input: "data source", + output: "output destination", + "input-and-output": "data source and output destination", +}; + +const QUERY_ACTIONS = "logs:StartQuery and logs:GetQueryResults"; +const WRITE_ACTIONS = "logs:CreateLogGroup, logs:CreateLogStream and logs:PutLogEvents"; +const NEEDED: Record = { + input: QUERY_ACTIONS, + output: WRITE_ACTIONS, + "input-and-output": `${QUERY_ACTIONS}, plus ${WRITE_ACTIONS}`, +}; + export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "update", @@ -23,6 +39,12 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => flag("id", "the ID of the online evaluation config to update", z.string().optional(), { group: "Target:", }), + flag( + "description", + "replace the description of the config's monitoring purpose", + z.string().optional(), + { group: "Configuration:" }, + ), flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { group: SESSION_SOURCE, }), @@ -66,6 +88,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "replace the IAM role the online evaluation assumes", @@ -74,7 +97,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => ), flag( "update-role", - "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", + "whether to re-scope an auto-provisioned execution role when the data source or output destination changes (default true)", z.enum(["true", "false"]).optional(), { group: EXECUTION }, ), @@ -97,9 +120,12 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => } const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], io); const { response, roleScopeWarning } = await core.eval.updateOnlineEvaluationConfig( flags["id"], { + description: flags["description"], + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( @@ -123,7 +149,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => // Suppressed under --json, matching runtime/invoke's advisory summary: a // scripted caller gets a machine-readable stdout and nothing else. if (roleScopeWarning && !ctx.require(JsonKey)) { - const { reason, roleArn, logGroupNames } = roleScopeWarning; + const { reason, roleArn, scope, logGroupNames } = roleScopeWarning; if (reason === "stale-scope") { // The update succeeded and the role grants the new data source; the // policy for the superseded one just could not be detached. @@ -138,9 +164,9 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => ? "it is not managed by the CLI" : "re-scoping was declined via --update-role false"; io.stderr.write( - `warning: the data source moved but the execution role was not re-scoped because ${detail}.\n` + + `warning: the ${MOVED[scope]} moved but the execution role was not re-scoped because ${detail}.\n` + ` role: ${roleArn}\n` + - ` ensure it grants logs:StartQuery and logs:GetQueryResults on: ${logGroupNames.join(", ")}\n`, + ` ensure it grants ${NEEDED[scope]} on: ${logGroupNames.join(", ")}\n`, ); } } diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 7c8dd4c29..79e5e4afb 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -28,6 +28,7 @@ import type { UpdateConfigurationBundleResponse, UpdateEvaluatorResponse, UpdateOnlineEvaluationConfigResponse, + OutputConfig as OnlineEvalOutputConfig, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { CreateABTestResponse, @@ -157,6 +158,8 @@ export type CreateOnlineEvalInput = { evaluatorIds?: string[]; evaluationExecutionRoleArn?: string; enableOnCreate?: boolean; + tags?: Record; + outputConfig?: OnlineEvalOutputConfig; } & ( | { agent: string; endpoint?: string; dataSourceConfig?: undefined } | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } @@ -204,6 +207,7 @@ export type DeleteOnlineInsightResponse = DeleteOnlineEvaluationConfigResponse; // `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to // the agent's default log group. export type UpdateOnlineEvalInput = { + description?: string; samplingRate?: number; sessionTimeoutMinutes?: number; filters?: Rule["filters"]; @@ -219,21 +223,22 @@ export type UpdateOnlineEvalInput = { // Replaces the execution role. The CLI never edits the permissions of a role the // caller names here — it is theirs to manage. evaluationExecutionRoleArn?: string; - // Whether to re-scope a CLI-provisioned role when the data source moves - // (default true). Only meaningful for a managed role: the old policy grants - // query access to the previous log groups only. + outputConfig?: OnlineEvalOutputConfig; updateRole?: boolean; }; // RoleScopeWarning reports that an execution role was left scoped to log groups -// the config no longer samples, so the caller can surface it. Returned rather -// than logged from Core so the handler owns how it is presented. export type RoleScopeWarning = { reason: "custom-role" | "update-declined" | "stale-scope"; roleArn: string; + scope: RoleScopeKind; logGroupNames: string[]; }; +export type RoleScopeKind = "input" | "output" | "input-and-output"; + +export type { OnlineEvalOutputConfig }; + export type BundleRef = { configBundle: string; bundleVersion: string }; export type CreateConfigBasedABTestInput = { diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index cf3bdd18e..df13f5e8b 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -185,6 +185,7 @@ import type { StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, + RoleScopeWarning, } from "../handlers/eval/types"; import { isTerminalStatus } from "../core/batchEvaluationResults"; import { abortable } from "../core/abortable"; @@ -1532,6 +1533,7 @@ export class TestEvalClient implements CoreEvalClient { DEFAULT_CREATE_ONLINE_EVAL_RESPONSE; private onlineEvalUpdateResponse: UpdateOnlineEvaluationConfigResponse = DEFAULT_UPDATE_ONLINE_EVAL_RESPONSE; + private onlineEvalRoleScopeWarning: RoleScopeWarning | undefined; private onlineEvalGetResponse: GetOnlineEvaluationConfigResponse = DEFAULT_GET_ONLINE_EVAL_RESPONSE; private onlineEvalDeleteResponse: DeleteOnlineEvaluationConfigResponse = @@ -1668,6 +1670,11 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setOnlineEvalRoleScopeWarning(warning: RoleScopeWarning): this { + this.onlineEvalRoleScopeWarning = warning; + return this; + } + // setOnlineEvalGetResponse sets what getOnlineEvaluationConfig resolves to // (when not erroring). setOnlineEvalGetResponse(response: GetOnlineEvaluationConfigResponse): this { @@ -2122,10 +2129,16 @@ export class TestEvalClient implements CoreEvalClient { id: string, update: UpdateOnlineEvalInput, options: CoreOptions, - ): Promise<{ response: UpdateOnlineEvaluationConfigResponse }> { + ): Promise<{ + response: UpdateOnlineEvaluationConfigResponse; + roleScopeWarning?: RoleScopeWarning; + }> { this.calls.push({ method: "updateOnlineEvaluationConfig", args: [id, update, options] }); if (this.error) throw this.error; - return { response: this.onlineEvalUpdateResponse }; + return { + response: this.onlineEvalUpdateResponse, + roleScopeWarning: this.onlineEvalRoleScopeWarning, + }; } async getOnlineEvaluationConfig(