From f47bc2fcfe2631b3c660f98ff9cb81bc0d2a839a Mon Sep 17 00:00:00 2001 From: Saeed Kasmani Date: Sun, 9 Aug 2026 21:14:45 +1000 Subject: [PATCH] Add Qwen3 VL model support --- README.md | 10 ++ deploy.sh | 5 +- deployment/DEPLOYMENT_README.md | 53 ++++++++ deployment/app.py | 33 +++-- .../foundation/bedrock_client.py | 116 ++++++++++++++++-- .../image_enhancer/agentic_enhancer.py | 6 +- deployment/runtime/agent/main-websocket.py | 19 +-- deployment/s3_files/S3_FILES_README.md | 2 +- .../agentcore_runtime_websocket_stack.py | 46 +++++-- deployment/stacks/iam_stack.py | 2 + deployment/stacks/lambda_stack.py | 9 +- 11 files changed, 258 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index a6fbf53..1693ce7 100644 --- a/README.md +++ b/README.md @@ -482,6 +482,16 @@ Model ID to environment variable mapping: | `*claude-opus-4-6*` | `CLAUDE_OPUS_PROFILE_ARN` | | `*nova-premier*` | `NOVA_PREMIER_PROFILE_ARN` | +### Qwen3 VL + +BADGERS supports the multimodal Bedrock model +`qwen.qwen3-vl-235b-a22b` for the agent and specialists. Qwen uses direct +regional invocation and the Bedrock Converse API. Do not configure Claude +thinking fields for Qwen. See the +[deployment model configuration](deployment/DEPLOYMENT_README.md#qwen3-vl-support) +for agent and specialist examples and verify that the model is available in your +deployment Region. + ### ➕ Adding a New Specialist **Option 1: Use the Wizard (Recommended)** diff --git a/deploy.sh b/deploy.sh index f01cedb..0b9c192 100755 --- a/deploy.sh +++ b/deploy.sh @@ -109,10 +109,13 @@ step_infra() { "$(_sn DynamoDB)" "$(_sn IAM)" "$(_sn ECR)" - "$(_sn InferenceProfiles)" "$(_sn Memory)" "$(_sn Vpc)" ) + if [ "${BADGERS_SKIP_INFERENCE_PROFILES:-}" != "1" ] \ + && [[ "${BADGERS_MODEL_ID:-}" != qwen.* ]]; then + stacks+=("$(_sn InferenceProfiles)") + fi if [ "${BADGERS_SKIP_XRAY}" != "1" ]; then stacks+=("$(_sn XRay)") fi diff --git a/deployment/DEPLOYMENT_README.md b/deployment/DEPLOYMENT_README.md index fdd4066..c83b6ec 100644 --- a/deployment/DEPLOYMENT_README.md +++ b/deployment/DEPLOYMENT_README.md @@ -469,6 +469,59 @@ Each specialist has a manifest file in `s3_files/manifests/` that configures its > [!NOTE] > Extended thinking is only supported on Claude models. When enabled, thinking content is saved to S3 alongside results: `{session_id}/{specialist_name}/{image}_thinking_{timestamp}.txt` +### Qwen3 VL support + +`qwen.qwen3-vl-235b-a22b` is supported for both the orchestrating agent and +image-aware specialists through the Amazon Bedrock Converse API. Confirm that the +model is available in the deployment Region before selecting it. + +To use Qwen3 VL for the orchestrating agent, update +`s3_files/agent_config/agent_model_config.json` before deployment (or update the +same object in the configuration bucket): + +```json +{ + "model_id": "qwen.qwen3-vl-235b-a22b", + "temperature": 0.1, + "max_tokens": 8000, + "fallback_models": [] +} +``` + +To use it for a specialist, select the same model ID in that specialist's +manifest: + +```json +"model_selections": { + "primary": { + "model_id": "qwen.qwen3-vl-235b-a22b" + }, + "fallback_list": [] +} +``` + +Qwen does not use the Claude `thinking`, `extended_thinking`, +`adaptive_thinking`, or `budget_tokens` fields. Omit those fields from Qwen +configurations. The deployment grants direct regional invocation permission for +this model. When `BADGERS_MODEL_ID` starts with `qwen.`, the application inference +profile stack is omitted automatically because it is not required and its +profile-backed models are not available in every Region. Set +`BADGERS_SKIP_INFERENCE_PROFILES=1` to request the same direct-invocation-only +deployment explicitly. + +For a deployment-time override of the agent and image-enhancer models, export +the model IDs before running the deployment command: + +```bash +export BADGERS_MODEL_ID=qwen.qwen3-vl-235b-a22b +export BADGERS_VISION_MODEL_ID=qwen.qwen3-vl-235b-a22b +./deploy.sh +``` + +These variables configure the orchestrating agent and the image enhancer. Each +specialist still follows its own manifest, allowing Qwen adoption per specialist +without changing unrelated model selections. + Simple format (no extended thinking) is still supported for backward compatibility: ```json "model_selections": { diff --git a/deployment/app.py b/deployment/app.py index e1155a7..060a0e7 100644 --- a/deployment/app.py +++ b/deployment/app.py @@ -167,15 +167,26 @@ def _sn(name: str) -> str: description="ECR repository for AgentCore Runtime agent container", ) -# Inference Profiles for cost tracking -inference_profiles_stack = InferenceProfilesStack( - app, - _sn("InferenceProfiles"), - deployment_id=deployment_id, - deployment_tags=deployment_tags, - env=env, - description="Application Inference Profiles for cost tracking and usage monitoring", +# Application inference profiles are only required by profile-backed models. Qwen is +# invoked directly in the deployment Region, and some Regions do not offer the +# cross-Region profiles configured by this stack. +model_id = os.environ.get("BADGERS_MODEL_ID", "").strip().lower() +SKIP_INFERENCE_PROFILES = ( + os.environ.get("BADGERS_SKIP_INFERENCE_PROFILES", "").strip() == "1" + or model_id.startswith("qwen.") ) +inference_profiles_stack = None +if SKIP_INFERENCE_PROFILES: + print("Application inference profiles omitted for direct model invocation") +else: + inference_profiles_stack = InferenceProfilesStack( + app, + _sn("InferenceProfiles"), + deployment_id=deployment_id, + deployment_tags=deployment_tags, + env=env, + description="Application Inference Profiles for cost tracking and usage monitoring", + ) # Load deployment config for selective specialist deployment enabled_specialists = None @@ -212,7 +223,8 @@ def _sn(name: str) -> str: description="Lambda specialists for BADGERS", ) lambda_stack.add_dependency(ecr_stack) -lambda_stack.add_dependency(inference_profiles_stack) +if inference_profiles_stack is not None: + lambda_stack.add_dependency(inference_profiles_stack) lambda_stack.add_dependency(dynamodb_stack) # X-Ray Transaction Search (account-level prerequisite for AgentCore tracing). @@ -291,7 +303,8 @@ def _sn(name: str) -> str: runtime_websocket_stack.add_dependency(gateway_stack) runtime_websocket_stack.add_dependency(cognito_stack) runtime_websocket_stack.add_dependency(memory_stack) -runtime_websocket_stack.add_dependency(inference_profiles_stack) +if inference_profiles_stack is not None: + runtime_websocket_stack.add_dependency(inference_profiles_stack) if xray_stack is not None: runtime_websocket_stack.add_dependency(xray_stack) runtime_websocket_stack.add_dependency(dynamodb_stack) diff --git a/deployment/badgers-foundation/foundation/bedrock_client.py b/deployment/badgers-foundation/foundation/bedrock_client.py index 138d716..1d0d457 100644 --- a/deployment/badgers-foundation/foundation/bedrock_client.py +++ b/deployment/badgers-foundation/foundation/bedrock_client.py @@ -4,6 +4,7 @@ Handles Bedrock client creation, invocation, and error handling for different specialist types. """ +import base64 import json import logging import time @@ -81,7 +82,7 @@ def get_model_family(model_id: str) -> str: model_id: The Bedrock model ID Returns: - 'claude' or 'nova' + 'claude', 'nova', or 'qwen' Raises: BedrockError: If model family cannot be determined @@ -92,6 +93,8 @@ def get_model_family(model_id: str) -> str: return "claude" elif "nova" in model_lower or "amazon.nova" in model_lower: return "nova" + elif "qwen" in model_lower: + return "qwen" else: raise BedrockError(f"Unknown model family for model ID: {model_id}") @@ -365,6 +368,20 @@ def _invoke_single_model( adaptive_thinking, ) + if model_family == "qwen": + if extended_thinking or adaptive_thinking: + self.logger.warning( + "Extended/adaptive thinking is not supported for Qwen models; ignoring" + ) + normalized = self._invoke_qwen_with_converse( + client, + invoke_model_id, + payload, + max_retries=max_retries, + ) + normalized["model_id"] = model_id + return normalized + # Convert payload to model-specific format model_payload = self._convert_payload_for_model( payload, @@ -396,6 +413,7 @@ def _invoke_single_model( # Normalize response to common format normalized = self._normalize_response(response_body, model_family) + normalized["model_id"] = model_id self.logger.info("Model invocation successful") return normalized @@ -405,6 +423,76 @@ def _invoke_single_model( except Exception as e: raise BedrockError(f"Model invocation failed: {e}") from e + def _invoke_qwen_with_converse( + self, + client, + model_id: str, + payload: Dict[str, Any], + max_retries: int = 3, + ) -> Dict[str, Any]: + """Invoke a Qwen model with the Bedrock Converse API.""" + messages: list[Dict[str, Any]] = [] + for message in payload.get("messages", []): + content = message.get("content", []) + if isinstance(content, str): + content = [{"type": "text", "text": content}] + + converted_content: list[Dict[str, Any]] = [] + for item in content: + if item.get("type") == "text" or "text" in item: + converted_content.append({"text": item.get("text", "")}) + elif item.get("type") == "image": + source = item.get("source", {}) + image_format = ( + source.get("media_type", "image/png") + .split("/", 1)[-1] + .lower() + ) + if image_format == "jpg": + image_format = "jpeg" + image_data = source.get("data", "") + if isinstance(image_data, str): + image_data = base64.b64decode(image_data) + converted_content.append( + { + "image": { + "format": image_format, + "source": {"bytes": image_data}, + } + } + ) + + messages.append( + { + "role": message.get("role", "user"), + "content": converted_content, + } + ) + + converse_args: Dict[str, Any] = { + "modelId": model_id, + "messages": messages, + "inferenceConfig": { + "maxTokens": min(int(payload.get("max_tokens", 8000)), 8000), + "temperature": float(payload.get("temperature", 0.1)), + }, + } + system_prompt = payload.get("system") + if system_prompt: + converse_args["system"] = ( + [{"text": system_prompt}] + if isinstance(system_prompt, str) + else system_prompt + ) + + time.sleep(self.throttle_delay) + response = self.handle_throttling( + client.converse, + max_retries=max_retries, + **converse_args, + ) + return self._normalize_response(response, "qwen") + def _convert_payload_for_model( self, payload: Dict[str, Any], @@ -419,7 +507,7 @@ def _convert_payload_for_model( Args: payload: Base payload with system, messages, max_tokens, temperature - model_family: 'claude' or 'nova' + model_family: 'claude', 'nova', or 'qwen' extended_thinking: Whether to enable extended thinking (Claude only) budget_tokens: Optional budget tokens for extended thinking adaptive_thinking: Whether to enable adaptive thinking (Claude only) @@ -442,6 +530,8 @@ def _convert_payload_for_model( "Extended/adaptive thinking not supported for Nova models, ignoring" ) return self._convert_to_nova_payload(payload) + elif model_family == "qwen": + return payload else: raise BedrockError(f"Unknown model family: {model_family}") @@ -576,7 +666,7 @@ def _normalize_response( Args: response_body: Raw response from model - model_family: 'claude' or 'nova' + model_family: 'claude', 'nova', or 'qwen' Returns: Normalized response with 'content' key and optional 'thinking' key @@ -600,22 +690,28 @@ def _normalize_response( return result - elif model_family == "nova": - # Convert Nova response to Claude format + elif model_family in {"nova", "qwen"}: + # Convert Bedrock Converse/Nova response to Claude-compatible content. try: nova_content = response_body["output"]["message"]["content"] - # Convert Nova content format to Claude format claude_content = [] for item in nova_content: if "text" in item: claude_content.append({"type": "text", "text": item["text"]}) if not claude_content: - raise BedrockError("Empty response from Nova model") + raise BedrockError(f"Empty response from {model_family} model") - return {"content": claude_content} + result = {"content": claude_content} + if "usage" in response_body: + result["usage"] = response_body["usage"] + if "stopReason" in response_body: + result["stop_reason"] = response_body["stopReason"] + return result except KeyError as e: - raise BedrockError(f"Invalid Nova response structure: {e}") from e + raise BedrockError( + f"Invalid {model_family} response structure: {e}" + ) from e else: raise BedrockError(f"Unknown model family: {model_family}") @@ -749,6 +845,8 @@ def validate_model_id(self, model_id: str) -> bool: "amazon.nova-", "us.amazon.nova-", "global.amazon.nova-", + # Qwen + "qwen.", # Other supported models "amazon.titan-", "ai21.j2-", diff --git a/deployment/lambdas/containers/image_enhancer/agentic_enhancer.py b/deployment/lambdas/containers/image_enhancer/agentic_enhancer.py index ebe313c..de15bf8 100644 --- a/deployment/lambdas/containers/image_enhancer/agentic_enhancer.py +++ b/deployment/lambdas/containers/image_enhancer/agentic_enhancer.py @@ -35,8 +35,10 @@ AWS_REGION = os.environ.get("AWS_REGION", "us-west-2") # Use application inference profile ARN for cost tracking and cross-region routing # Falls back to system inference profile ID if not set -VISION_MODEL = os.environ.get( - "CLAUDE_OPUS_46_PROFILE_ARN", "us.anthropic.claude-opus-4-6-v1" +VISION_MODEL = ( + os.environ.get("VISION_MODEL") + or os.environ.get("CLAUDE_OPUS_46_PROFILE_ARN") + or "us.anthropic.claude-opus-4-6-v1" ) MAX_ITERATIONS = int(os.environ.get("MAX_ITERATIONS", "2")) MAX_IMAGE_DIMENSION = int(os.environ.get("MAX_IMAGE_DIMENSION", "4000")) diff --git a/deployment/runtime/agent/main-websocket.py b/deployment/runtime/agent/main-websocket.py index 2882280..b637f6c 100644 --- a/deployment/runtime/agent/main-websocket.py +++ b/deployment/runtime/agent/main-websocket.py @@ -504,13 +504,18 @@ async def stream_agent_events( RUNTIME SESSION ID: {runtime_session_id} Include session_id: "{runtime_session_id}" in ALL tool calls.""" - model = BedrockModel( - model_id=model_config.get("model_id", DEFAULT_MODEL_CONFIG["model_id"]), - region_name=os.environ.get("AWS_REGION", "us-west-2"), - temperature=model_config.get("temperature", 1.0), - max_tokens=model_config.get("max_tokens", 8000), - additional_request_fields={"thinking": model_config.get("thinking", {})}, - ) + model_kwargs: dict[str, Any] = { + "model_id": os.environ.get("BADGERS_MODEL_ID") + or model_config.get("model_id", DEFAULT_MODEL_CONFIG["model_id"]), + "region_name": os.environ.get("AWS_REGION", "us-west-2"), + "temperature": model_config.get("temperature", 1.0), + "max_tokens": model_config.get("max_tokens", 8000), + } + thinking = model_config.get("thinking") + if thinking: + model_kwargs["additional_request_fields"] = {"thinking": thinking} + + model = BedrockModel(**model_kwargs) mcp_client = MCPClient(lambda: create_mcp_transport(gateway_url, access_token)) diff --git a/deployment/s3_files/S3_FILES_README.md b/deployment/s3_files/S3_FILES_README.md index f974b77..7e95389 100644 --- a/deployment/s3_files/S3_FILES_README.md +++ b/deployment/s3_files/S3_FILES_README.md @@ -28,7 +28,7 @@ Contains configuration for the orchestrating agent that coordinates PDF analysis | File | Purpose | | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| `agent_model_config.json` | Model selection (Claude Sonnet 4.5), temperature, max tokens, and thinking budget configuration | +| `agent_model_config.json` | Agent model selection, temperature, max tokens, and optional thinking configuration | | `agent_operating_environment_config.json` | ⚠️ **Critical** — Operating environment context injected into all prompts (agent + specialists). See below. | ### ⚠️ Operating Environment Configuration diff --git a/deployment/stacks/agentcore_runtime_websocket_stack.py b/deployment/stacks/agentcore_runtime_websocket_stack.py index 0979bdb..73fc1b7 100644 --- a/deployment/stacks/agentcore_runtime_websocket_stack.py +++ b/deployment/stacks/agentcore_runtime_websocket_stack.py @@ -3,7 +3,8 @@ Separate runtime stack for WebSocket streaming support. """ -from typing import TYPE_CHECKING +import os +from typing import TYPE_CHECKING, Optional from aws_cdk import ( Stack, @@ -45,7 +46,7 @@ def __init__( source_bucket_name: str, memory_id: str, s3_kms_key_arn: str, - inference_profiles_stack: "InferenceProfilesStack", + inference_profiles_stack: Optional["InferenceProfilesStack"], jobs_table: dynamodb.ITable, image_tag: str = "websocket", **kwargs, @@ -74,8 +75,9 @@ def __init__( self.agent_role = self.create_agent_role() - # Grant inference profile permissions via CDK grants - self.inference_profiles_stack.grant_invoke_to_role(self.agent_role) + # Grant profile-backed model permissions when profiles are deployed. + if self.inference_profiles_stack is not None: + self.inference_profiles_stack.grant_invoke_to_role(self.agent_role) self.runtime = self.create_runtime(ecr_image_uri) @@ -221,7 +223,19 @@ def create_agent_role(self) -> iam.Role: ) ) - # Note: Bedrock permissions are granted via inference_profiles_stack.grant_invoke_to_role() + role.add_to_policy( + iam.PolicyStatement( + sid="InvokeQwenFoundationModel", + effect=iam.Effect.ALLOW, + actions=[ + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + resources=[ + f"arn:aws:bedrock:{self.region}::foundation-model/qwen.qwen3-vl-235b-a22b" + ], + ) + ) role.add_to_policy( iam.PolicyStatement( @@ -423,12 +437,22 @@ def create_runtime(self, ecr_image_uri: str) -> agentcore.CfnRuntime: # foundation.job_state reads this at call time; when it is absent # every write becomes a no-op and tracking is simply off. "JOBS_TABLE_NAME": self.jobs_table.table_name, - # Inference profile ARNs for cost tracking - "CLAUDE_SONNET_PROFILE_ARN": self.inference_profiles_stack.claude_sonnet_profile_arn, - "CLAUDE_HAIKU_PROFILE_ARN": self.inference_profiles_stack.claude_haiku_profile_arn, - "NOVA_PREMIER_PROFILE_ARN": self.inference_profiles_stack.nova_premier_profile_arn, - "CLAUDE_OPUS_46_PROFILE_ARN": self.inference_profiles_stack.claude_opus_46_profile_arn, - "CLAUDE_OPUS_45_PROFILE_ARN": self.inference_profiles_stack.claude_opus_45_profile_arn, + **( + { + "CLAUDE_SONNET_PROFILE_ARN": self.inference_profiles_stack.claude_sonnet_profile_arn, + "CLAUDE_HAIKU_PROFILE_ARN": self.inference_profiles_stack.claude_haiku_profile_arn, + "NOVA_PREMIER_PROFILE_ARN": self.inference_profiles_stack.nova_premier_profile_arn, + "CLAUDE_OPUS_46_PROFILE_ARN": self.inference_profiles_stack.claude_opus_46_profile_arn, + "CLAUDE_OPUS_45_PROFILE_ARN": self.inference_profiles_stack.claude_opus_45_profile_arn, + } + if self.inference_profiles_stack is not None + else {} + ), + **( + {"BADGERS_MODEL_ID": os.environ["BADGERS_MODEL_ID"]} + if os.environ.get("BADGERS_MODEL_ID") + else {} + ), }, ) diff --git a/deployment/stacks/iam_stack.py b/deployment/stacks/iam_stack.py index 647a587..524f3f0 100644 --- a/deployment/stacks/iam_stack.py +++ b/deployment/stacks/iam_stack.py @@ -122,6 +122,8 @@ def __init__( "arn:aws:bedrock:*::foundation-model/anthropic.claude-opus-4-6-v1", # Claude Sonnet 4 foundation model (cell grid resolver) "arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-4-20250514-v1:0", + # Qwen multimodal model (direct regional invocation) + f"arn:aws:bedrock:{self.region}::foundation-model/qwen.qwen3-vl-235b-a22b", ], ) ) diff --git a/deployment/stacks/lambda_stack.py b/deployment/stacks/lambda_stack.py index dce6e37..28b4fad 100644 --- a/deployment/stacks/lambda_stack.py +++ b/deployment/stacks/lambda_stack.py @@ -4,6 +4,7 @@ import json import logging +import os from pathlib import Path from typing import TYPE_CHECKING, Optional from aws_cdk import ( @@ -375,8 +376,12 @@ def _create_ecr_container_function(self, func_name: str) -> lambda_.Function: } ) - # Image enhancer uses VISION_MODEL to select its model - point it at the application inference profile - if func_name == "image_enhancer": + # Image enhancer can use a direct model even when profile creation is skipped. + if func_name == "image_enhancer": + vision_model = os.environ.get("BADGERS_VISION_MODEL_ID") + if vision_model: + environment["VISION_MODEL"] = vision_model + elif self.inference_profiles_stack: environment["VISION_MODEL"] = ( self.inference_profiles_stack.claude_sonnet_46_profile_arn )