diff --git a/bun.lock b/bun.lock index a99d76eea..b254f4aa7 100644 --- a/bun.lock +++ b/bun.lock @@ -7,8 +7,8 @@ "dependencies": { "@aws-cdk/toolkit-lib": "1.38.2", "@aws-sdk/client-bedrock-agent": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0", + "@aws-sdk/client-bedrock-agentcore": "^3.1129.0", + "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0", "@aws-sdk/client-cloudformation": "^3.1092.0", "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", "@aws-sdk/client-iam": "^3.1080.0", @@ -95,9 +95,9 @@ "@aws-sdk/client-bedrock-agent": ["@aws-sdk/client-bedrock-agent@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-RAjn1g6X+u4WwnuwxYU8sKna5q1waTBCM+kzXWAvxSaNbgRsXFL6ZylPjOl0qCDZ7x6aVTj7Jb7Q0z6DZifP3w=="], - "@aws-sdk/client-bedrock-agentcore": ["@aws-sdk/client-bedrock-agentcore@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-VxagAO73X0efH+5oQfOqWc2/q1wTkiZ1IkBg7L0GnPBo4NFYw37H8klNbbrTqoluMAzRiBwsHoGX2jfnjZlDOQ=="], + "@aws-sdk/client-bedrock-agentcore": ["@aws-sdk/client-bedrock-agentcore@3.1130.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-node": "^3.972.83", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-HgHR24kDJwRN9NNrtWNeuyrJZgl0nXOikeeD2HBAprsRXFl3tvZQw6hHOADHR41sjT4z3+SmqIqN5U7JiaF40Q=="], - "@aws-sdk/client-bedrock-agentcore-control": ["@aws-sdk/client-bedrock-agentcore-control@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-zta1MHik3+WbRUVwRFyFWSGTEfhaMPDDG+qdzmLAAyK3m7gqYHJr6+p+6dvBeiO6U15FQ1PWSjj2C3O5ikiyAw=="], + "@aws-sdk/client-bedrock-agentcore-control": ["@aws-sdk/client-bedrock-agentcore-control@3.1130.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-node": "^3.972.83", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-EB3zB2zmxVsx+ZqusNjwBHhsqzWlkMLwyO7TIvc9tWHh02eV0lkmRT3HE4L6+DvZrE9R2x6KBl4OSkfU1TZvGA=="], "@aws-sdk/client-cloudcontrol": ["@aws-sdk/client-cloudcontrol@3.1121.0", "", { "dependencies": { "@aws-sdk/core": "^3.977.9", "@aws-sdk/credential-provider-node": "^3.972.81", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-kpR+c528XFWFT04APYPKrhSCyzdEJPgxryR0wWs2dfIj7Ovqwak+1zLwFOS8zpNqDPEx8xn7j3dqA/HTCo+qLw=="], @@ -1531,6 +1531,14 @@ "@aws-crypto/util/@smithy/util-utf8": ["@smithy/util-utf8@2.3.0", "", { "dependencies": { "@smithy/util-buffer-from": "^2.2.0", "tslib": "^2.6.2" } }, "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A=="], + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/core": ["@aws-sdk/core@3.978.0", "", { "dependencies": { "@aws-sdk/types": "^3.974.5", "@aws-sdk/xml-builder": "^3.972.40", "@aws/lambda-invoke-store": "^0.3.0", "@smithy/core": "^3.33.3", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.17.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-2yX9LUmxPklVjSGTb8dfnWRJSiFQ3TeH2nn7G1mdKHTfnabzF0+gfrS8rYfLWmZrQ8A3mEcxMJjRc51dL5KWaA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.83", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-ini": "^3.973.16", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-jdso7ejzfRnatxMUZK4S/U6KbaDPCvfIV4XL+IQAPFDBt5rj5Fq595euqlK8Le4lNCMFR9oUpt+1l0aMgaayOQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/core": ["@aws-sdk/core@3.978.0", "", { "dependencies": { "@aws-sdk/types": "^3.974.5", "@aws-sdk/xml-builder": "^3.972.40", "@aws/lambda-invoke-store": "^0.3.0", "@smithy/core": "^3.33.3", "@smithy/signature-v4": "^5.6.12", "@smithy/types": "^4.17.2", "bowser": "^2.11.0", "tslib": "^2.6.2" } }, "sha512-2yX9LUmxPklVjSGTb8dfnWRJSiFQ3TeH2nn7G1mdKHTfnabzF0+gfrS8rYfLWmZrQ8A3mEcxMJjRc51dL5KWaA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node": ["@aws-sdk/credential-provider-node@3.972.83", "", { "dependencies": { "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-ini": "^3.973.16", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-jdso7ejzfRnatxMUZK4S/U6KbaDPCvfIV4XL+IQAPFDBt5rj5Fq595euqlK8Le4lNCMFR9oUpt+1l0aMgaayOQ=="], + "@aws-sdk/protocol-http/@smithy/protocol-http": ["@smithy/protocol-http@1.2.0", "", { "dependencies": { "@smithy/types": "^1.2.0", "tslib": "^2.5.0" } }, "sha512-GfGfruksi3nXdFok5RhgtOnWe5f6BndzYfmEXISD+5gAGdayFGpjWu5pIqIweTudMtse20bGbc+7MFZXT1Tb8Q=="], "@aws-sdk/signature-v4/@smithy/signature-v4": ["@smithy/signature-v4@1.1.0", "", { "dependencies": { "@smithy/eventstream-codec": "^1.1.0", "@smithy/is-array-buffer": "^1.1.0", "@smithy/types": "^1.2.0", "@smithy/util-hex-encoding": "^1.1.0", "@smithy/util-middleware": "^1.1.0", "@smithy/util-uri-escape": "^1.1.0", "@smithy/util-utf8": "^1.1.0", "tslib": "^2.5.0" } }, "sha512-fDo3m7YqXBs7neciOePPd/X9LPm5QLlDMdIC4m1H6dgNLnXfLMFNIxEfPyohGA8VW9Wn4X8lygnPSGxDZSmp0Q=="], @@ -1651,6 +1659,30 @@ "@aws-cdk/cx-api/@aws-cdk/cloud-assembly-schema/semver": ["semver@7.8.5", "", { "bundled": true, "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-JN+JHruYZw3GUZB8YGAlDk4wTDPOEAEEdEzj5nS0xodWR4smzHsN7PnK2j6IeOsDIj2aqua5DSbhXl9Gtf90FQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.73", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uyYYnJOnlis8uQzaYGPd7N1JoioCoNpXgnkXYixsWJXHXgXyYi8WXJSDfofxJeWfQIGWLe2Nwyq60Uc7MZdVOg=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.973.16", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-login": "^3.972.78", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-i++ly+0Uxa+u3ebSSyr0S/3CFhFJDxCXT3+Zj+mW2bXenEx5bKGCdTIKFu39SgXBNhWDjex/8cXUx9MUTMCrTw=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-lYmXJa4gvq4xN1lrT5NiP5vIYYKcGWAdj8y+8o6dlcateB5eF3Dn8DtmjjHKfMBrTPAMr2pebIiX/UOj8c1/UA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.973.15", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/token-providers": "3.1129.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-6Jhcf4v0pSFdjk1EW2kvzuEBKD+UZ2uNcHUIglKKLndD20YhvkL2kdmDOV5/j4mYuWWwe/a1FQ1aomU86/Cg5Q=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.77", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uylIQSUWpfLuH2LovxEEfwzJGM/SabLOfLMg6YXu/E8jJEKUdpdILCVCQCdFvHyu/7dLJOHPMfrSwduxO56NkQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-env": ["@aws-sdk/credential-provider-env@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-JN+JHruYZw3GUZB8YGAlDk4wTDPOEAEEdEzj5nS0xodWR4smzHsN7PnK2j6IeOsDIj2aqua5DSbhXl9Gtf90FQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-http": ["@aws-sdk/credential-provider-http@3.972.73", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uyYYnJOnlis8uQzaYGPd7N1JoioCoNpXgnkXYixsWJXHXgXyYi8WXJSDfofxJeWfQIGWLe2Nwyq60Uc7MZdVOg=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini": ["@aws-sdk/credential-provider-ini@3.973.16", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/credential-provider-env": "^3.972.71", "@aws-sdk/credential-provider-http": "^3.972.73", "@aws-sdk/credential-provider-login": "^3.972.78", "@aws-sdk/credential-provider-process": "^3.972.71", "@aws-sdk/credential-provider-sso": "^3.973.15", "@aws-sdk/credential-provider-web-identity": "^3.972.77", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/credential-provider-imds": "^4.4.16", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-i++ly+0Uxa+u3ebSSyr0S/3CFhFJDxCXT3+Zj+mW2bXenEx5bKGCdTIKFu39SgXBNhWDjex/8cXUx9MUTMCrTw=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-process": ["@aws-sdk/credential-provider-process@3.972.71", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-lYmXJa4gvq4xN1lrT5NiP5vIYYKcGWAdj8y+8o6dlcateB5eF3Dn8DtmjjHKfMBrTPAMr2pebIiX/UOj8c1/UA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso": ["@aws-sdk/credential-provider-sso@3.973.15", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/token-providers": "3.1129.0", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-6Jhcf4v0pSFdjk1EW2kvzuEBKD+UZ2uNcHUIglKKLndD20YhvkL2kdmDOV5/j4mYuWWwe/a1FQ1aomU86/Cg5Q=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity": ["@aws-sdk/credential-provider-web-identity@3.972.77", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-uylIQSUWpfLuH2LovxEEfwzJGM/SabLOfLMg6YXu/E8jJEKUdpdILCVCQCdFvHyu/7dLJOHPMfrSwduxO56NkQ=="], + "@aws-sdk/protocol-http/@smithy/protocol-http/@smithy/types": ["@smithy/types@1.2.0", "", { "dependencies": { "tslib": "^2.5.0" } }, "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA=="], "@aws-sdk/signature-v4/@smithy/signature-v4/@smithy/types": ["@smithy/types@1.2.0", "", { "dependencies": { "tslib": "^2.5.0" } }, "sha512-z1r00TvBqF3dh4aHhya7nz1HhvCg4TRmw51fjMrh5do3h+ngSstt/yKlNbHeb9QxJmFbmN8KEVSWgb1bRvfEoA=="], @@ -1691,6 +1723,26 @@ "@aws-cdk/cloudformation-diff/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.78", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-eUtswnXu0+Ii9ieRK+0L7aPFV3Z/dnW2VntJzjBP9xs8s+8p5nBNuymIXtXwZ+5r5+XJP3e32nMkuZ/r0HozEA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1129.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-Sbl3rpzQdsG4ZK2zh0JWUYyZPKKorJlVOddA2T0DVbKJFrsW8J6wgnslxxUH04+WaBMr4A1HzJZvZX0xUvkniA=="], + + "@aws-sdk/client-bedrock-agentcore-control/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/credential-provider-login": ["@aws-sdk/credential-provider-login@3.972.78", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-eUtswnXu0+Ii9ieRK+0L7aPFV3Z/dnW2VntJzjBP9xs8s+8p5nBNuymIXtXwZ+5r5+XJP3e32nMkuZ/r0HozEA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-ini/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-sso/@aws-sdk/token-providers": ["@aws-sdk/token-providers@3.1129.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/nested-clients": "^3.997.45", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-Sbl3rpzQdsG4ZK2zh0JWUYyZPKKorJlVOddA2T0DVbKJFrsW8J6wgnslxxUH04+WaBMr4A1HzJZvZX0xUvkniA=="], + + "@aws-sdk/client-bedrock-agentcore/@aws-sdk/credential-provider-node/@aws-sdk/credential-provider-web-identity/@aws-sdk/nested-clients": ["@aws-sdk/nested-clients@3.997.45", "", { "dependencies": { "@aws-sdk/core": "^3.978.0", "@aws-sdk/signature-v4-multi-region": "^3.996.46", "@aws-sdk/types": "^3.974.5", "@smithy/core": "^3.33.3", "@smithy/fetch-http-handler": "^5.7.2", "@smithy/node-http-handler": "^4.11.3", "@smithy/types": "^4.17.2", "tslib": "^2.6.2" } }, "sha512-mooq9Q+jLa18VoM7HouczmslZU60iiB0aKc/Ztnq/luIL1ud0z4DnYprLR/ZO1gp331S9tJctM1HZr7u6YKBXQ=="], + "@aws-sdk/signature-v4/@smithy/signature-v4/@smithy/util-utf8/@smithy/util-buffer-from": ["@smithy/util-buffer-from@1.1.0", "", { "dependencies": { "@smithy/is-array-buffer": "^1.1.0", "tslib": "^2.5.0" } }, "sha512-9m6NXE0ww+ra5HKHCHig20T+FAwxBAm7DIdwc/767uGWbRcY720ybgPacQNB96JMOI7xVr/CDa3oMzKmW4a+kw=="], "@typescript-eslint/typescript-estree/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], diff --git a/package.json b/package.json index 5ab4d027e..c2b2792cf 100644 --- a/package.json +++ b/package.json @@ -64,8 +64,8 @@ "dependencies": { "@aws-cdk/toolkit-lib": "1.38.2", "@aws-sdk/client-bedrock-agent": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore": "^3.1092.0", - "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0", + "@aws-sdk/client-bedrock-agentcore": "^3.1129.0", + "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0", "@aws-sdk/client-cloudformation": "^3.1092.0", "@aws-sdk/client-cloudwatch-logs": "^3.1092.0", "@aws-sdk/client-iam": "^3.1080.0", diff --git a/src/core/eval.test.ts b/src/core/eval.test.ts new file mode 100644 index 000000000..8917393e0 --- /dev/null +++ b/src/core/eval.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from "bun:test"; +import { + StartBatchEvaluationCommand, + type BedrockAgentCoreClient, +} from "@aws-sdk/client-bedrock-agentcore"; +import type { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import type { IAMClient } from "@aws-sdk/client-iam"; +import { CoreClient } from "./index"; +import { createSilentLogger } from "../testing"; +import type { OutputConfig } from "@aws-sdk/client-bedrock-agentcore"; + +const OPTIONS = { region: "us-west-2" }; + +const RAW_SOURCE = { + origin: "raw" as const, + dataSourceConfig: { + cloudWatchLogs: { serviceNames: ["my_agent.DEFAULT"], logGroupNames: ["/some/group"] }, + }, +}; + +function coreWithCapturedDataCommands() { + const sent: unknown[] = []; + const unusable = (name: string) => () => { + throw new Error(`this test should not construct the ${name} client`); + }; + const core = new CoreClient({ + createDataClient: () => + ({ + send: async (command: unknown) => { + sent.push(command); + return { batchEvaluationId: "batch-eval-1", status: "IN_PROGRESS" }; + }, + }) as unknown as BedrockAgentCoreClient, + createControlClient: unusable("control") as unknown as () => BedrockAgentCoreControlClient, + createIamClient: unusable("IAM") as unknown as () => IAMClient, + createLogsClient: unusable("logs") as unknown as () => CloudWatchLogsClient, + logger: createSilentLogger(), + }); + const startInput = () => { + const command = sent.find((c) => c instanceof StartBatchEvaluationCommand); + expect(command).toBeDefined(); + return (command as StartBatchEvaluationCommand).input; + }; + return { core, startInput }; +} + +describe("CoreClient.eval.startBatchEvaluation", () => { + test("forwards outputConfig to the request unchanged", async () => { + const outputConfig = { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + metricsNamespace: "Company/AgentEvaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + } as OutputConfig; + const { core, startInput } = coreWithCapturedDataCommands(); + + await core.eval.startBatchEvaluation( + { name: "eval-1", evaluatorIds: ["Builtin.Helpfulness"], source: RAW_SOURCE, outputConfig }, + OPTIONS, + ); + + expect(startInput().outputConfig).toEqual(outputConfig); + }); + + test("omits outputConfig when the caller did not supply one", async () => { + const { core, startInput } = coreWithCapturedDataCommands(); + + await core.eval.startBatchEvaluation( + { name: "eval-1", evaluatorIds: ["Builtin.Helpfulness"], source: RAW_SOURCE }, + OPTIONS, + ); + + expect(startInput().outputConfig).toBeUndefined(); + }); +}); diff --git a/src/core/eval.tsx b/src/core/eval.tsx index bf9080d51..28b417c3c 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -123,6 +123,8 @@ import type { DatasetUpdateProgressEvent, DatasetUpdateResult, RoleScopeWarning, + RoleScopeKind, + OnlineEvalOutputConfig, CoreEvalClient, CreateConfigurationBundleInput, CreateConfigBasedABTestInput, @@ -679,6 +681,7 @@ export class EvalClient implements CoreEvalClient { dataSourceConfig, evaluationMetadata: input.groundTruth ? { sessionMetadata: input.groundTruth } : undefined, kmsKeyArn: input.kmsKeyArn, + outputConfig: input.outputConfig, }), ); } @@ -988,6 +991,9 @@ export class EvalClient implements CoreEvalClient { options.region, logGroupNamesOf(dataSourceConfig), await evaluatorKmsKeys(input.evaluatorIds ?? [], control), + // Read only to widen the write scope to the chosen destination; the + // request object below still gets the caller's object untouched. + { outputConfig: input.outputConfig }, ) ).roleArn; @@ -997,8 +1003,10 @@ export class EvalClient implements CoreEvalClient { rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), dataSourceConfig, evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + outputConfig: input.outputConfig, evaluationExecutionRoleArn, enableOnCreate: input.enableOnCreate ?? true, + tags: input.tags, }); // A role provisioned moments ago may not be assumable yet (IAM is eventually @@ -1236,6 +1244,11 @@ export class EvalClient implements CoreEvalClient { ? dataSourceConfig : undefined; + const outputMoved = update.outputConfig !== undefined; + const effectiveOutputConfig = update.outputConfig ?? current.outputConfig; + const scopeKind: RoleScopeKind = + movedTo !== undefined && outputMoved ? "input-and-output" : outputMoved ? "output" : "input"; + const configName = current.onlineEvaluationConfigName; const roleArn = update.evaluationExecutionRoleArn ?? current.evaluationExecutionRoleArn; const managedRoleName = @@ -1245,26 +1258,33 @@ export class EvalClient implements CoreEvalClient { isManagedOnlineEvalRole(roleArn, configName) ? configName : undefined; - const refreshManagedRole = movedTo !== undefined && managedRoleName !== undefined; - - if (movedTo !== undefined && managedRoleName === undefined && roleArn) { + const scopeChanged = movedTo !== undefined || outputMoved; + const refreshManagedRole = scopeChanged && managedRoleName !== undefined; + const affectedLogGroups = [ + ...(movedTo !== undefined ? logGroupNamesOf(movedTo) : []), + ...(outputMoved ? destinationLogGroupNames(update.outputConfig, dataSourceConfig) : []), + ]; + + if (scopeChanged && managedRoleName === undefined && roleArn) { roleScopeWarning = { reason: "custom-role", roleArn, - logGroupNames: logGroupNamesOf(movedTo), + scope: scopeKind, + logGroupNames: affectedLogGroups, }; - } else if (movedTo !== undefined && !refreshManagedRole && roleArn) { + } else if (scopeChanged && !refreshManagedRole && roleArn) { // managed role, but the caller declined the refresh roleScopeWarning = { reason: "update-declined", roleArn, - logGroupNames: logGroupNamesOf(movedTo), + scope: scopeKind, + logGroupNames: affectedLogGroups, }; } if (refreshManagedRole && update.updateRole !== false) { const iam = this.clients.iam({ region: options.region }); - const newLogGroups = logGroupNamesOf(movedTo); + const newLogGroups = dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : []; const oldLogGroups = current.dataSourceConfig ? logGroupNamesOf(current.dataSourceConfig) : []; @@ -1289,7 +1309,7 @@ export class EvalClient implements CoreEvalClient { options.region, newLogGroups, kmsKeys, - resourceNameFromArn(roleArn!), + { roleName: resourceNameFromArn(roleArn!), outputConfig: effectiveOutputConfig }, ); const oldPolicyName = scopePolicyName( executionPolicy( @@ -1297,15 +1317,18 @@ export class EvalClient implements CoreEvalClient { accountIdFromRoleArn(managedRoleArn), oldLogGroups, kmsKeys, + current.outputConfig, ), ); const response = await control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, + description: update.description, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), dataSourceConfig, evaluators, + outputConfig: update.outputConfig, }), ); @@ -1322,6 +1345,7 @@ export class EvalClient implements CoreEvalClient { roleScopeWarning = { reason: "stale-scope", roleArn: roleArn!, + scope: scopeKind, logGroupNames: oldLogGroups, }; } @@ -1332,9 +1356,11 @@ export class EvalClient implements CoreEvalClient { const response = await control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, + description: update.description, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), dataSourceConfig, evaluators, + outputConfig: update.outputConfig, evaluationExecutionRoleArn: update.evaluationExecutionRoleArn, }), ); @@ -2200,6 +2226,18 @@ function logGroupNamesOf(dataSourceConfig: DataSourceConfig): string[] { : []; } +function destinationLogGroupNames( + outputConfig: OnlineEvalOutputConfig | undefined, + dataSourceConfig: DataSourceConfig | undefined, +): string[] { + const cloudWatch = outputConfig?.cloudWatchConfig; + if (!cloudWatch) return []; + if (cloudWatch.resultDestination === "SOURCE_LOG_GROUP") { + return dataSourceConfig ? logGroupNamesOf(dataSourceConfig) : []; + } + return cloudWatch.logGroupName ? [cloudWatch.logGroupName] : []; +} + // runtimeIdFromLogGroup recovers the runtime id embedded in a log group path // produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for a // new --endpoint without the caller passing --agent again. Returns undefined for diff --git a/src/core/onlineEvalExecutionRole.test.ts b/src/core/onlineEvalExecutionRole.test.ts index 7f1ec01ea..e4a719410 100644 --- a/src/core/onlineEvalExecutionRole.test.ts +++ b/src/core/onlineEvalExecutionRole.test.ts @@ -110,3 +110,68 @@ test("gives identical policies the same name", () => { scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])), ); }); + +function writeStatement(policy: string) { + return statements(policy).find((s) => s.Sid === "WriteEvaluationResults"); +} + +const SERVICE_RESULTS = `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/evaluations/*`; + +test("a config with no output destination keeps the service namespace as a bare string", () => { + const write = writeStatement(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])); + + expect(write?.Resource).toBe(SERVICE_RESULTS); + expect(Array.isArray(write?.Resource)).toBe(false); +}); + +test("a customer-named dedicated group is granted alongside the service namespace", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/company/agent-evaluations*`, + ]); + expect(write?.Action).toContain("logs:CreateLogGroup"); +}); + +test("SOURCE_LOG_GROUP grants writes to the groups the traces are read from", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" }, + }), + ); + + expect(write?.Resource).toEqual([ + SERVICE_RESULTS, + `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`, + ]); +}); + +test("a destination already inside the service namespace adds nothing", () => { + const write = writeStatement( + executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { + logGroupName: "/aws/bedrock-agentcore/evaluations/online-evaluations/results/default", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }), + ); + + expect(write?.Resource).toBe(SERVICE_RESULTS); +}); + +test("changing the destination changes the policy name, so a re-scope is a new grant", () => { + const before = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, []); + const after = executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [], { + cloudWatchConfig: { logGroupName: "/company/agent-evaluations" }, + }); + + expect(scopePolicyName(after)).not.toBe(scopePolicyName(before)); +}); diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx index e02a6f40a..4c1896b6e 100644 --- a/src/core/onlineEvalExecutionRole.tsx +++ b/src/core/onlineEvalExecutionRole.tsx @@ -15,7 +15,6 @@ import { parseArn, resourceNameFromArn } from "./arn"; // evaluation results back to CloudWatch. When the caller doesn't bring one, // OnlineEvalClient provisions a per-config default here, scoped to the log // group(s) being sampled. Idempotent: an existing role is reused. -// // Each scope is stored as its own inline policy, named after a fingerprint of the // scope, so granting a new scope never overwrites the policy backing the current // one. IAM unions Allows across a role's inline policies, which lets an update @@ -83,10 +82,35 @@ function runtimeLogGroupPrefix(logGroupName: string): string { return match?.[1] ?? logGroupName; } +const SERVICE_RESULT_PREFIX = "/aws/bedrock-agentcore/evaluations/"; + +function resultWriteArns( + logs: string, + sampledArns: string[], + outputConfig: OnlineEvalResultDestination | undefined, +): string | string[] { + const arns = [`${logs}:${SERVICE_RESULT_PREFIX}*`]; + const cloudWatch = outputConfig?.cloudWatchConfig; + + if (cloudWatch?.resultDestination === "SOURCE_LOG_GROUP") { + arns.push(...sampledArns); + } else if ( + cloudWatch?.logGroupName && + !cloudWatch.logGroupName.startsWith(SERVICE_RESULT_PREFIX) + ) { + arns.push(`${logs}:${cloudWatch.logGroupName}*`); + } + + return arns.length === 1 ? arns[0]! : arns; +} + +export type OnlineEvalResultDestination = { + cloudWatchConfig?: { logGroupName?: string; resultDestination?: string } | undefined; +}; + // executionPolicy grants the permissions CreateOnlineEvaluationConfig validates // at creation time. Exported for assertion: the policy body is not observable // through the recorded IAM fixtures, whose responses are empty. -// // at creation time: Logs Insights query access over the sampled log groups plus // the `aws/spans` group that carries the actual trace spans, Bedrock model // invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based @@ -98,6 +122,7 @@ export function executionPolicy( accountId: string, logGroupNames: string[], kmsKeyArns: string[], + outputConfig?: OnlineEvalResultDestination, ): string { const logs = `arn:aws:logs:${region}:${accountId}:log-group`; const spansArn = `${logs}:aws/spans`; @@ -134,6 +159,8 @@ export function executionPolicy( Resource: [`${spansArn}*`, ...sampledArns], }, { + // logs:CreateLogGroup is needed because the service creates a + // customer-named result group that does not exist yet. Sid: "WriteEvaluationResults", Effect: "Allow", Action: [ @@ -142,7 +169,7 @@ export function executionPolicy( "logs:DescribeLogStreams", "logs:PutLogEvents", ], - Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`, + Resource: resultWriteArns(logs, sampledArns, outputConfig), }, { Sid: "IndexSpans", @@ -201,6 +228,11 @@ export function scopePolicyName(policyDocument: string): string { return `${POLICY_PREFIX}-${fingerprint(policyDocument)}`; } +export type GrantScopeOptions = { + roleName?: string; + outputConfig?: OnlineEvalResultDestination; +}; + // grantOnlineEvalScope creates the execution role for `configName` if it does not // exist and attaches the inline policy for this scope, returning the role ARN and // the policy name written. The caller revokes the superseded scope once whatever @@ -211,7 +243,7 @@ export async function grantOnlineEvalScope( region: string, logGroupNames: string[], kmsKeyArns: string[] = [], - roleName = onlineEvalExecutionRoleName(configName), + { roleName = onlineEvalExecutionRoleName(configName), outputConfig }: GrantScopeOptions = {}, ): Promise<{ roleArn: string; policyName: string }> { let roleArn: string; try { @@ -234,6 +266,7 @@ export async function grantOnlineEvalScope( accountIdFromRoleArn(roleArn), logGroupNames, kmsKeyArns, + outputConfig, ); const policyName = scopePolicyName(policyDocument); await iam.send( diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index 98d49f7b2..9170a30b5 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -34,10 +34,14 @@ const RESULTS: BatchEvaluationResultEntry[] = [ { evaluatorId: "Builtin.Helpfulness", level: "Session", sessionId: "s1", score: 5 }, ]; -async function run(args: string[], configure?: (core: TestCoreClient) => void) { +async function run( + args: string[], + configure?: (core: TestCoreClient) => void, + ioOptions?: { stdin?: string }, +) { const core = new TestCoreClient(); configure?.(core); - const io = testIO(); + const io = testIO(ioOptions); const root = createRootHandler(core, { io: io.io, logger: createSilentLogger(), @@ -262,3 +266,139 @@ describe("eval batch-evaluation simulate", () => { expect(JSON.parse(stdout).failures).toEqual([{ exampleId: "bad", error: "HTTP 500" }]); }); }); + +describe("eval batch-evaluation evaluate --output-config", () => { + const BASE = [ + "eval", + "batch-evaluation", + "evaluate", + "--agent", + "r-1", + "--evaluators", + "Builtin.Helpfulness", + "--name", + "eval-1", + ]; + const CONFIG = { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + metricsNamespace: "Company/AgentEvaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, + }; + + function startInput(core: TestCoreClient) { + const call = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect(call).toBeDefined(); + return call!.args[0] as { outputConfig?: unknown }; + } + + test("reaches the request unchanged from inline JSON", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(CONFIG)]); + expect(startInput(core).outputConfig).toEqual(CONFIG); + }); + + test("reaches the request unchanged from a file", async () => { + const path = `${process.env["TMPDIR"] ?? "/tmp"}/agentcore-output-config-${Bun.hash(JSON.stringify(CONFIG))}.json`; + await Bun.write(path, JSON.stringify(CONFIG)); + const { core } = await run([...BASE, "--output-config", `file://${path}`]); + expect(startInput(core).outputConfig).toEqual(CONFIG); + }); + + test("reaches the request unchanged from stdin", async () => { + const { core } = await run([...BASE, "--output-config", "-"], undefined, { + stdin: JSON.stringify(CONFIG), + }); + expect(startInput(core).outputConfig).toEqual(CONFIG); + }); + + test("is left undefined when omitted, so the service keeps its default destination", async () => { + const { core } = await run(BASE); + expect(startInput(core).outputConfig).toBeUndefined(); + }); + + test("rejects malformed JSON before any SDK call", async () => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route([ + "node", + "agentcore", + ...BASE, + "--output-config", + "{not json", + "--region", + "us-west-2", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + expect(core.eval.calls).toEqual([]); + }); +}); + +describe("eval batch-evaluation simulate --endpoint and --output-config", () => { + const BASE = [ + "eval", + "batch-evaluation", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluators", + "Builtin.Helpfulness", + "--name", + "sim-1", + ]; + const invoked = (c: TestCoreClient) => + c.eval.setInvokeDatasetResponse({ + sessions: [{ exampleId: "e1", sessionId: "s1" }], + invoked: 1, + failed: 0, + failures: [], + }); + + test("--endpoint drives both the Runtime invocation and the graded session source", async () => { + const { core } = await run([...BASE, "--endpoint", "BETA"], invoked); + const invoke = core.eval.calls.find((c) => c.method === "invokeDataset"); + expect((invoke!.args[0] as { qualifier?: string }).qualifier).toBe("BETA"); + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect((start!.args[0] as { source: { endpoint?: string } }).source.endpoint).toBe("BETA"); + }); + + test("malformed --output-config aborts before any Runtime is invoked", async () => { + const core = new TestCoreClient(); + invoked(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route([ + "node", + "agentcore", + ...BASE, + "--output-config", + "{not json", + "--region", + "us-west-2", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + expect(core.eval.calls.map((c) => c.method)).not.toContain("invokeDataset"); + }); + + test("a valid --output-config reaches the graded job", async () => { + const config = { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }; + const { core } = await run([...BASE, "--output-config", JSON.stringify(config)], invoked); + const start = core.eval.calls.find((c) => c.method === "startBatchEvaluation"); + expect((start!.args[0] as { outputConfig?: unknown }).outputConfig).toEqual(config); + }); +}); diff --git a/src/handlers/eval/batch-evaluation/evaluate/index.tsx b/src/handlers/eval/batch-evaluation/evaluate/index.tsx index 8b95678ad..17e9d92f8 100644 --- a/src/handlers/eval/batch-evaluation/evaluate/index.tsx +++ b/src/handlers/eval/batch-evaluation/evaluate/index.tsx @@ -7,6 +7,7 @@ import type { Core } from "../../../types"; import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; import { coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { SessionSource } from "../../sessionSource"; +import { BatchOutputConfig } from "../outputConfig"; const CONFIGURATION = "Configuration:"; const EVALUATION = "Evaluation:"; @@ -59,6 +60,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => z.string().optional(), { group: EVALUATION, help: groundTruthHelp }, ), + ...BatchOutputConfig.flags, ], handle: async (ctx, flags) => { if (!flags["name"]) { @@ -78,6 +80,8 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => await resolver.resolveText("ground-truth", flags["ground-truth"]), ); + const outputConfig = await BatchOutputConfig.resolve(flags["output-config"], io); + const response = await core.eval.startBatchEvaluation( { name: flags["name"], @@ -86,6 +90,7 @@ export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) => source, groundTruth, kmsKeyArn: flags["kms-key-arn"], + outputConfig, }, coreOptsFromCtx(ctx), ); diff --git a/src/handlers/eval/batch-evaluation/outputConfig.tsx b/src/handlers/eval/batch-evaluation/outputConfig.tsx new file mode 100644 index 000000000..d3fead6b1 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/outputConfig.tsx @@ -0,0 +1,54 @@ +import type { OutputConfig } from "@aws-sdk/client-bedrock-agentcore"; +import z from "zod"; +import { SourceResolver, type AppIO } from "../../../io"; +import { flag } from "../../../router"; +import { parseJsonFlag } from "../../utils"; + +const outputConfigHelp = `(JSON: tagged union object) +Where evaluation results and metrics are written. Omit it and results go to the +service-managed default location. Only top-level key: cloudWatchConfig. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchConfig": { + "logGroupName": "string", // result log group; omit for + // SOURCE_LOG_GROUP, and it cannot sit + // under /aws/bedrock-agentcore/evaluations/ + "logStreamName": "string", // result log stream + "metricsNamespace": "string", // defaults to + // Bedrock-AgentCore/Evaluations; cannot + // begin with "AWS/" + "resultDestination": "DEDICATED_LOG_GROUP" | "SOURCE_LOG_GROUP" + // DEDICATED_LOG_GROUP (default) writes to + // a dedicated result group; + // SOURCE_LOG_GROUP writes back to the log + // group the traces were read from + } + } + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore/latest/APIReference/API_OutputConfig.html + +Example: + --output-config '{"cloudWatchConfig":{"logGroupName":"/company/agent-evaluations","metricsNamespace":"Company/AgentEvaluations","resultDestination":"DEDICATED_LOG_GROUP"}}'`; + +export class BatchOutputConfig { + static readonly flags = [ + flag( + "output-config", + "where results and metrics are written (JSON OutputConfig)", + z.string().optional(), + { group: "Result output:", help: outputConfigHelp }, + ), + ] as const; + + static async resolve(value: string | undefined, io: AppIO): Promise { + const resolver = new SourceResolver({ stdin: io.stdin }); + return parseJsonFlag( + "output-config", + await resolver.resolveText("output-config", value), + ); + } +} diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx index ff6c51ec7..23ee6d69c 100644 --- a/src/handlers/eval/batch-evaluation/simulate/index.tsx +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -6,6 +6,7 @@ import type { AppIO } from "../../../../io"; import type { Core } from "../../../types"; import { coreOptsFromCtx } from "../../../utils"; import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; +import { BatchOutputConfig } from "../outputConfig"; const RUNTIME_INVOCATION = "Runtime invocation:"; const DATASET = "Dataset:"; @@ -23,7 +24,7 @@ Example: // Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror // `runtime invoke`. -export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => +export const createSimulateBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ name: "simulate", description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions", @@ -31,7 +32,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), { group: RUNTIME_INVOCATION, }), - flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { + flag("endpoint", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), { group: RUNTIME_INVOCATION, }), flag( @@ -77,6 +78,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), { group: "Evaluation:", }), + ...BatchOutputConfig.flags, ], handle: async (ctx, flags) => { if (!flags["runtime-id"]) @@ -96,6 +98,8 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). // TODO(#1986): swap for the shared SIGINT/abort helper once it merges. + const outputConfig = await BatchOutputConfig.resolve(flags["output-config"], io); + const controller = new AbortController(); const interrupt = () => controller.abort(); process.once("SIGINT", interrupt); @@ -105,7 +109,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => const r = await core.eval.invokeDataset( { runtimeId: flags["runtime-id"], - qualifier: flags["qualifier"], + qualifier: flags["endpoint"], payloadTemplate: flags["payload-template"], headers: parseRuntimeInvokeHeaders(flags["header"]), bearerToken: flags["bearer-token"], @@ -134,7 +138,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => source: { origin: "agent", agent: flags["runtime-id"], - endpoint: flags["qualifier"], + endpoint: flags["endpoint"], sessionIds: r.sessions.map((s) => s.sessionId), }, groundTruth: r.sessions.map((s) => ({ @@ -143,6 +147,7 @@ export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => ...(s.groundTruth && { groundTruth: { inline: s.groundTruth } }), })), kmsKeyArn: flags["kms-key-arn"], + outputConfig, }, opts, ); diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index 273b95c7b..c28b7bb49 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -5,9 +5,24 @@ import { InputValidationError } from "../../../../errors"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; -import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; +import { + assertMutuallyExclusiveFlags, + coreOptsFromCtx, + parseJsonFlag, + parseJsonFlagWithSchema, +} from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; +import { TagsSchema } from "../../../../projectSchemas/tags"; + +const tagsHelp = `(JSON: map of string to string) +Tags applied to the online evaluation configuration. + +Accepts inline JSON, file://, or - to read stdin. + +Example: + --tags '{"team":"ml-platform","env":"prod"}'`; const CONFIGURATION = "Configuration:"; const SESSION_SOURCE = "Session source (choose exactly one):"; @@ -35,6 +50,10 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => z.enum(["true", "false"]).optional(), { group: CONFIGURATION }, ), + flag("tags", "resource tags (JSON object of key/value strings)", z.string().optional(), { + group: CONFIGURATION, + help: tagsHelp, + }), flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional(), { group: SESSION_SOURCE, }), @@ -69,6 +88,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "IAM role the online evaluation assumes (default auto-provisioned)", @@ -98,9 +118,17 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => } const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], io); + const tags = parseJsonFlagWithSchema( + "tags", + await source.resolveText("tags", flags["tags"]), + TagsSchema, + ); const common = { name: flags["name"], description: flags["description"], + tags, + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( diff --git a/src/handlers/eval/online-eval/online-eval.flags.test.tsx b/src/handlers/eval/online-eval/online-eval.flags.test.tsx new file mode 100644 index 000000000..e198386ce --- /dev/null +++ b/src/handlers/eval/online-eval/online-eval.flags.test.tsx @@ -0,0 +1,178 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { createSilentLogger, TestCoreClient, testIO } from "../../../testing"; +import { TestGlobalConfigAccessor } from "../../../testing/"; +import type { CreateOnlineEvalInput, UpdateOnlineEvalInput } from "../types"; + +async function run( + args: string[], + configure?: (core: TestCoreClient) => void, + ioOptions?: { stdin?: string }, +) { + const core = new TestCoreClient(); + configure?.(core); + const io = testIO(ioOptions); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +function createInput(core: TestCoreClient): CreateOnlineEvalInput { + const call = core.eval.calls.find((c) => c.method === "createOnlineEvaluationConfig"); + expect(call).toBeDefined(); + return call!.args[0] as CreateOnlineEvalInput; +} + +function updateInput(core: TestCoreClient): UpdateOnlineEvalInput { + const call = core.eval.calls.find((c) => c.method === "updateOnlineEvaluationConfig"); + expect(call).toBeDefined(); + return call!.args[1] as UpdateOnlineEvalInput; +} + +const OUTPUT_CONFIG = { + cloudWatchConfig: { + logGroupName: "/company/agent-evaluations", + metricsNamespace: "Company/AgentEvaluations", + resultDestination: "DEDICATED_LOG_GROUP", + }, +} as const; + +describe("eval online-eval create", () => { + const BASE = [ + "eval", + "online-eval", + "create", + "--name", + "quality", + "--agent", + "r-1", + "--evaluators", + "Builtin.Helpfulness", + "--sampling-rate", + "10", + ]; + + test("--output-config reaches Core unchanged from inline JSON", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(OUTPUT_CONFIG)]); + expect(createInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("--output-config reaches Core unchanged from stdin", async () => { + const { core } = await run([...BASE, "--output-config", "-"], undefined, { + stdin: JSON.stringify(OUTPUT_CONFIG), + }); + expect(createInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("--tags reaches Core as a parsed map", async () => { + const { core } = await run([...BASE, "--tags", '{"team":"ml-platform","env":"dev"}']); + expect(createInput(core).tags).toEqual({ team: "ml-platform", env: "dev" }); + }); + + test("both are left undefined when omitted, so the service keeps its defaults", async () => { + const { core } = await run(BASE); + expect(createInput(core).outputConfig).toBeUndefined(); + expect(createInput(core).tags).toBeUndefined(); + }); + + test.each([ + ["--output-config", "{not json", /Invalid JSON for option '--output-config'/], + ["--tags", "{not json", /Invalid JSON for option '--tags'/], + ])("malformed %s fails before Core provisions anything", async (flag, value, expected) => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route(["node", "agentcore", ...BASE, flag, value, "--region", "us-west-2"]), + ).rejects.toThrow(expected); + expect(core.eval.calls).toEqual([]); + }); + + test("--tags rejects a non-string value rather than passing it to the API", async () => { + await expect(run([...BASE, "--tags", '{"team":42}'])).rejects.toThrow( + /Invalid value for option '--tags'/, + ); + }); +}); + +describe("eval online-eval update", () => { + const BASE = ["eval", "online-eval", "update", "--id", "online-eval-123"]; + + test("--description reaches Core", async () => { + const { core } = await run([...BASE, "--description", "checks tone on prod traffic"]); + expect(updateInput(core).description).toBe("checks tone on prod traffic"); + }); + + test("--output-config reaches Core unchanged", async () => { + const { core } = await run([...BASE, "--output-config", JSON.stringify(OUTPUT_CONFIG)]); + expect(updateInput(core).outputConfig).toEqual(OUTPUT_CONFIG); + }); + + test("an omitted --output-config is not sent, so the destination is preserved", async () => { + const { core } = await run([...BASE, "--sampling-rate", "20"]); + expect(updateInput(core).outputConfig).toBeUndefined(); + }); + + test("malformed --output-config fails before the initial Get", async () => { + const core = new TestCoreClient(); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await expect( + root.route([ + "node", + "agentcore", + ...BASE, + "--output-config", + "{not json", + "--region", + "us-west-2", + ]), + ).rejects.toThrow(/Invalid JSON for option '--output-config'/); + expect(core.eval.calls).toEqual([]); + }); + + test.each([ + ["input", "data source moved", "logs:StartQuery and logs:GetQueryResults"], + ["output", "output destination moved", "logs:PutLogEvents"], + ["input-and-output", "data source and output destination moved", "logs:PutLogEvents"], + ] as const)( + "a %s scope warning names what moved and what to grant", + async (scope, movedText, action) => { + const { stderr } = await run(BASE, (c) => + c.eval.setOnlineEvalRoleScopeWarning({ + reason: "custom-role", + roleArn: "arn:aws:iam::123456789012:role/MyRole", + scope, + logGroupNames: ["/company/agent-evaluations"], + }), + ); + expect(stderr).toContain(movedText); + expect(stderr).toContain(action); + expect(stderr).toContain("/company/agent-evaluations"); + }, + ); + + test("the scope warning is suppressed under --json", async () => { + const { stderr } = await run([...BASE, "--json"], (c) => + c.eval.setOnlineEvalRoleScopeWarning({ + reason: "custom-role", + roleArn: "arn:aws:iam::123456789012:role/MyRole", + scope: "output", + logGroupNames: ["/company/agent-evaluations"], + }), + ); + expect(stderr).toBe(""); + }); +}); diff --git a/src/handlers/eval/online-eval/outputConfig.tsx b/src/handlers/eval/online-eval/outputConfig.tsx new file mode 100644 index 000000000..b958592ae --- /dev/null +++ b/src/handlers/eval/online-eval/outputConfig.tsx @@ -0,0 +1,58 @@ +import z from "zod"; +import { SourceResolver, type AppIO } from "../../../io"; +import { flag } from "../../../router"; +import { parseJsonFlag } from "../../utils"; +import type { OnlineEvalOutputConfig } from "../types"; + +const outputConfigHelp = `(JSON object) +Where evaluation results and metrics are written. Omit it and results go to the +service-managed default location. Only top-level key: cloudWatchConfig. + +Accepts inline JSON, file://, or - to read stdin. + +JSON syntax: + { + "cloudWatchConfig": { + "logGroupName": "string", // result log group; omit for + // SOURCE_LOG_GROUP, and it cannot sit + // under /aws/bedrock-agentcore/evaluations/ + "metricsNamespace": "string", // CloudWatch metrics namespace + "resultDestination": "DEDICATED_LOG_GROUP" | "SOURCE_LOG_GROUP" + // DEDICATED_LOG_GROUP writes to a + // dedicated result group, creating it if + // needed; SOURCE_LOG_GROUP writes back to + // the groups the traces were sampled from + } + } + +A role you supply with --role-arn is never edited by the CLI, so it must already +grant logs:PutLogEvents on the destination — plus logs:CreateLogGroup when the +group does not exist yet. + +API reference: + https://docs.aws.amazon.com/bedrock-agentcore-control/latest/APIReference/API_OutputConfig.html + +Example: + --output-config '{"cloudWatchConfig":{"logGroupName":"/company/agent-evaluations","metricsNamespace":"Company/AgentEvaluations","resultDestination":"DEDICATED_LOG_GROUP"}}'`; + +export class OnlineEvalOutputConfigFlag { + static readonly flags = [ + flag( + "output-config", + "where results and metrics are written (JSON OutputConfig)", + z.string().optional(), + { group: "Result output:", help: outputConfigHelp }, + ), + ] as const; + + static async resolve( + value: string | undefined, + io: AppIO, + ): Promise { + const resolver = new SourceResolver({ stdin: io.stdin }); + return parseJsonFlag( + "output-config", + await resolver.resolveText("output-config", value), + ); + } +} diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 6bdaa6807..57d932e30 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -6,15 +6,31 @@ import { JsonKey } from "../../../keys"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; +import type { RoleScopeKind } from "../../types"; import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; import { filtersHelp } from "../filtersHelp"; import { onlineEvalDataSourceConfigHelp } from "../dataSourceConfigHelp"; +import { OnlineEvalOutputConfigFlag } from "../outputConfig"; const SESSION_SOURCE = "Session source:"; const SOURCE_FILTERS = "Source filters:"; const EVALUATION = "Evaluation:"; const EXECUTION = "Execution:"; +const MOVED: Record = { + input: "data source", + output: "output destination", + "input-and-output": "data source and output destination", +}; + +const QUERY_ACTIONS = "logs:StartQuery and logs:GetQueryResults"; +const WRITE_ACTIONS = "logs:CreateLogGroup, logs:CreateLogStream and logs:PutLogEvents"; +const NEEDED: Record = { + input: QUERY_ACTIONS, + output: WRITE_ACTIONS, + "input-and-output": `${QUERY_ACTIONS}, plus ${WRITE_ACTIONS}`, +}; + export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => createHandler({ name: "update", @@ -23,6 +39,12 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => flag("id", "the ID of the online evaluation config to update", z.string().optional(), { group: "Target:", }), + flag( + "description", + "replace the description of the config's monitoring purpose", + z.string().optional(), + { group: "Configuration:" }, + ), flag("agent", "repoint at a different harness ID or Runtime ID", z.string().optional(), { group: SESSION_SOURCE, }), @@ -66,6 +88,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => group: EVALUATION, help: filtersHelp, }), + ...OnlineEvalOutputConfigFlag.flags, flag( "role-arn", "replace the IAM role the online evaluation assumes", @@ -74,7 +97,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => ), flag( "update-role", - "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", + "whether to re-scope an auto-provisioned execution role when the data source or output destination changes (default true)", z.enum(["true", "false"]).optional(), { group: EXECUTION }, ), @@ -97,9 +120,12 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => } const source = new SourceResolver({ stdin: io.stdin }); + const outputConfig = await OnlineEvalOutputConfigFlag.resolve(flags["output-config"], io); const { response, roleScopeWarning } = await core.eval.updateOnlineEvaluationConfig( flags["id"], { + description: flags["description"], + outputConfig, samplingRate: flags["sampling-rate"], sessionTimeoutMinutes: flags["session-timeout-minutes"], filters: parseJsonFlag( @@ -123,7 +149,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => // Suppressed under --json, matching runtime/invoke's advisory summary: a // scripted caller gets a machine-readable stdout and nothing else. if (roleScopeWarning && !ctx.require(JsonKey)) { - const { reason, roleArn, logGroupNames } = roleScopeWarning; + const { reason, roleArn, scope, logGroupNames } = roleScopeWarning; if (reason === "stale-scope") { // The update succeeded and the role grants the new data source; the // policy for the superseded one just could not be detached. @@ -138,9 +164,9 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => ? "it is not managed by the CLI" : "re-scoping was declined via --update-role false"; io.stderr.write( - `warning: the data source moved but the execution role was not re-scoped because ${detail}.\n` + + `warning: the ${MOVED[scope]} moved but the execution role was not re-scoped because ${detail}.\n` + ` role: ${roleArn}\n` + - ` ensure it grants logs:StartQuery and logs:GetQueryResults on: ${logGroupNames.join(", ")}\n`, + ` ensure it grants ${NEEDED[scope]} on: ${logGroupNames.join(", ")}\n`, ); } } diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 8d84c3656..79e5e4afb 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -28,6 +28,7 @@ import type { UpdateConfigurationBundleResponse, UpdateEvaluatorResponse, UpdateOnlineEvaluationConfigResponse, + OutputConfig as OnlineEvalOutputConfig, } from "@aws-sdk/client-bedrock-agentcore-control"; import type { CreateABTestResponse, @@ -51,6 +52,7 @@ import type { InlineGroundTruth, EvaluationReferenceInput, EvaluationResultContent, + OutputConfig, DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; @@ -156,6 +158,8 @@ export type CreateOnlineEvalInput = { evaluatorIds?: string[]; evaluationExecutionRoleArn?: string; enableOnCreate?: boolean; + tags?: Record; + outputConfig?: OnlineEvalOutputConfig; } & ( | { agent: string; endpoint?: string; dataSourceConfig?: undefined } | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } @@ -203,6 +207,7 @@ export type DeleteOnlineInsightResponse = DeleteOnlineEvaluationConfigResponse; // `rule` object); `clearEndpoint` nulls out the endpoint scope, falling back to // the agent's default log group. export type UpdateOnlineEvalInput = { + description?: string; samplingRate?: number; sessionTimeoutMinutes?: number; filters?: Rule["filters"]; @@ -218,21 +223,22 @@ export type UpdateOnlineEvalInput = { // Replaces the execution role. The CLI never edits the permissions of a role the // caller names here — it is theirs to manage. evaluationExecutionRoleArn?: string; - // Whether to re-scope a CLI-provisioned role when the data source moves - // (default true). Only meaningful for a managed role: the old policy grants - // query access to the previous log groups only. + outputConfig?: OnlineEvalOutputConfig; updateRole?: boolean; }; // RoleScopeWarning reports that an execution role was left scoped to log groups -// the config no longer samples, so the caller can surface it. Returned rather -// than logged from Core so the handler owns how it is presented. export type RoleScopeWarning = { reason: "custom-role" | "update-declined" | "stale-scope"; roleArn: string; + scope: RoleScopeKind; logGroupNames: string[]; }; +export type RoleScopeKind = "input" | "output" | "input-and-output"; + +export type { OnlineEvalOutputConfig }; + export type BundleRef = { configBundle: string; bundleVersion: string }; export type CreateConfigBasedABTestInput = { @@ -289,6 +295,7 @@ export type StartBatchEvaluationInput = { // Already-parsed --ground-truth (SessionMetadataShape[]) → evaluationMetadata. groundTruth?: SessionMetadataShape[]; kmsKeyArn?: string; + outputConfig?: OutputConfig; }; // Batch insights use the same service job API as batch evaluations, but remain diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index cf3bdd18e..df13f5e8b 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -185,6 +185,7 @@ import type { StartRecommendationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, + RoleScopeWarning, } from "../handlers/eval/types"; import { isTerminalStatus } from "../core/batchEvaluationResults"; import { abortable } from "../core/abortable"; @@ -1532,6 +1533,7 @@ export class TestEvalClient implements CoreEvalClient { DEFAULT_CREATE_ONLINE_EVAL_RESPONSE; private onlineEvalUpdateResponse: UpdateOnlineEvaluationConfigResponse = DEFAULT_UPDATE_ONLINE_EVAL_RESPONSE; + private onlineEvalRoleScopeWarning: RoleScopeWarning | undefined; private onlineEvalGetResponse: GetOnlineEvaluationConfigResponse = DEFAULT_GET_ONLINE_EVAL_RESPONSE; private onlineEvalDeleteResponse: DeleteOnlineEvaluationConfigResponse = @@ -1668,6 +1670,11 @@ export class TestEvalClient implements CoreEvalClient { return this; } + setOnlineEvalRoleScopeWarning(warning: RoleScopeWarning): this { + this.onlineEvalRoleScopeWarning = warning; + return this; + } + // setOnlineEvalGetResponse sets what getOnlineEvaluationConfig resolves to // (when not erroring). setOnlineEvalGetResponse(response: GetOnlineEvaluationConfigResponse): this { @@ -2122,10 +2129,16 @@ export class TestEvalClient implements CoreEvalClient { id: string, update: UpdateOnlineEvalInput, options: CoreOptions, - ): Promise<{ response: UpdateOnlineEvaluationConfigResponse }> { + ): Promise<{ + response: UpdateOnlineEvaluationConfigResponse; + roleScopeWarning?: RoleScopeWarning; + }> { this.calls.push({ method: "updateOnlineEvaluationConfig", args: [id, update, options] }); if (this.error) throw this.error; - return { response: this.onlineEvalUpdateResponse }; + return { + response: this.onlineEvalUpdateResponse, + roleScopeWarning: this.onlineEvalRoleScopeWarning, + }; } async getOnlineEvaluationConfig(