From c62f092ac78e9ab7ad4474e7c7e564fe2a7a361e Mon Sep 17 00:00:00 2001 From: Saeed Kasmani Date: Sun, 9 Aug 2026 14:04:21 +1000 Subject: [PATCH] Add image input support --- .../agent_system_prompt.xml | 12 +++-- ui/UI_README.md | 7 +-- ui/server/routes/core.js | 45 ++++++++++++++++--- ui/src/components/Chat.jsx | 34 ++++++++++---- 4 files changed, 77 insertions(+), 21 deletions(-) diff --git a/deployment/s3_files/agent_system_prompt/agent_system_prompt.xml b/deployment/s3_files/agent_system_prompt/agent_system_prompt.xml index e024f48..576622d 100644 --- a/deployment/s3_files/agent_system_prompt/agent_system_prompt.xml +++ b/deployment/s3_files/agent_system_prompt/agent_system_prompt.xml @@ -1,7 +1,7 @@ - You are an intelligent PDF analysis orchestrator. You make all decisions about which tools to use based on content classification. You have full control over the analysis workflow. + You are an intelligent document and image analysis orchestrator. You make all decisions about which tools to use based on content classification. You have full control over the analysis workflow. @@ -47,8 +47,12 @@ - When user provides PDF path, call pdf_to_images_converter to convert PDF to images - Call classify_pdf_content_tool with the image paths to classify each page + + When the user provides a PDF path or S3 URI, call pdf_to_images_converter to convert every PDF page to images. + When the user provides a TIFF, TIF, JPEG, JPG, or PNG path or S3 URI, treat it as a single page image and DO NOT call pdf_to_images_converter. + Supported inputs are PDF, TIFF, TIF, JPEG, JPG, and PNG. + + Call classify_pdf_content_tool with a JSON array of image paths. For a direct image upload, pass a one-item array containing its path or S3 URI. Analyze the classification results and decide which specialist tools to use for each page. Check enhancement_recommended field for each page. For each page, call the appropriate specialist tool(s) based on classification. @@ -244,4 +248,4 @@ Minimal content page — full_text captures the page number. Elements is not necessary when there is no document structure to identify (no headings, no sections, just a page number). Correlator is not called since there is only one specialist result. - \ No newline at end of file + diff --git a/ui/UI_README.md b/ui/UI_README.md index 87f2844..9b054bd 100644 --- a/ui/UI_README.md +++ b/ui/UI_README.md @@ -88,7 +88,7 @@ Browser (React/Vite) ├── /api/* ──→ Express server (port 7860), all behind requireAuth │ ├── Core routes (all roles) │ │ ├── AgentCore WebSocket proxy (chat, SSE to browser) - │ │ ├── PDF upload to S3 (mints doc_id) + │ │ ├── PDF and image upload to S3 (mints doc_id) │ │ ├── Job records (/api/jobs) │ │ ├── S3 file operations (manifests, prompts, schemas) │ │ ├── CloudWatch Logs Insights queries @@ -112,7 +112,8 @@ The UI is the read side of the doc/job/subtask hierarchy described in | `GET /api/jobs/:jobId` | One job: computed status, counts, and every subtask with its outcome | | `GET /api/jobs?doc_id=` | Every job recorded against one document, newest first | -`POST /api/upload` mints a `doc_id` per upload and returns it. `POST /api/chat` accepts +`POST /api/upload` accepts PDF, PNG, JPEG/JPG, and TIFF/TIF files, validates their +file signatures, mints a `doc_id` per upload, and returns it. `POST /api/chat` accepts that `doc_id` and forwards it to the agent, which stamps it onto each specialist call. When the agent mints a `job_id` it emits a `job` SSE event, also written to the session log as `[job] job_id=… doc_id=…`, so a chat transcript can be traced to its job record. @@ -147,7 +148,7 @@ ui/ │ │ └── useUser.js # User context (role, email) from ID token claims │ └── components/ │ ├── Home.jsx # Dashboard -│ ├── Chat.jsx # Agent chat interface, PDF upload, doc_id +│ ├── Chat.jsx # Agent chat interface, PDF/image upload, doc_id │ ├── SpecialistEditor.jsx # Manifest/prompt editor │ ├── SpecialistWizard.jsx # New specialist wizard │ ├── Evaluator.jsx # Test runner diff --git a/ui/server/routes/core.js b/ui/server/routes/core.js index 822d4b2..bf9bec4 100644 --- a/ui/server/routes/core.js +++ b/ui/server/routes/core.js @@ -70,6 +70,30 @@ export function mountCoreRoutes(app, PROJECT_ROOT) { const ddbClient = DynamoDBDocumentClient.from(new DynamoDBClient({ region: REGION, credentials })); const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 50 * 1024 * 1024 } }); + function detectUploadType(buffer) { + if (buffer.length >= 5 && buffer.subarray(0, 5).toString() === '%PDF-') { + return { contentType: 'application/pdf', fileKind: 'pdf', extensions: ['.pdf'] }; + } + if (buffer.length >= 8 && buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) { + return { contentType: 'image/png', fileKind: 'image', extensions: ['.png'] }; + } + if (buffer.length >= 3 && buffer[0] === 0xff && buffer[1] === 0xd8 && buffer[2] === 0xff) { + return { contentType: 'image/jpeg', fileKind: 'image', extensions: ['.jpg', '.jpeg'] }; + } + const isClassicTiff = buffer.length >= 4 && ( + (buffer[0] === 0x49 && buffer[1] === 0x49 && buffer[2] === 0x2a && buffer[3] === 0x00) || + (buffer[0] === 0x4d && buffer[1] === 0x4d && buffer[2] === 0x00 && buffer[3] === 0x2a) + ); + const isBigTiff = buffer.length >= 4 && ( + (buffer[0] === 0x49 && buffer[1] === 0x49 && buffer[2] === 0x2b && buffer[3] === 0x00) || + (buffer[0] === 0x4d && buffer[1] === 0x4d && buffer[2] === 0x00 && buffer[3] === 0x2b) + ); + if (isClassicTiff || isBigTiff) { + return { contentType: 'image/tiff', fileKind: 'image', extensions: ['.tif', '.tiff'] }; + } + return null; + } + console.log(`Region: ${REGION}`); console.log(`Profile: ${AWS_PROFILE || 'default'}`); console.log(`Runtime ARN: ${RUNTIME_ARN ? '✅' : '❌ not set'}`); @@ -167,9 +191,13 @@ export function mountCoreRoutes(app, PROJECT_ROOT) { app.post('/api/upload', upload.single('file'), async (req, res) => { if (!req.file) return res.status(400).json({ error: 'No file provided' }); if (!UPLOAD_BUCKET) return res.status(500).json({ error: 'S3_UPLOAD_BUCKET not configured' }); - const isPdfMime = req.file.mimetype === 'application/pdf'; - const isPdfMagic = req.file.buffer.length >= 5 && req.file.buffer.slice(0, 5).toString() === '%PDF-'; - if (!isPdfMime || !isPdfMagic) return res.status(400).json({ error: 'Only PDF files are accepted' }); + const detectedType = detectUploadType(req.file.buffer); + const extension = req.file.originalname.toLowerCase().match(/\.[^.]+$/)?.[0]; + if (!detectedType || !detectedType.extensions.includes(extension)) { + return res.status(400).json({ + error: 'Supported formats: PDF, TIFF, TIF, JPEG, JPG, and PNG. File contents must match the extension.', + }); + } const filename = req.file.originalname.replace(/[^a-zA-Z0-9._-]/g, '_'); // doc_id is the top level of the job-tracking hierarchy @@ -182,10 +210,17 @@ export function mountCoreRoutes(app, PROJECT_ROOT) { Bucket: UPLOAD_BUCKET, Key: s3Key, Body: req.file.buffer, - ContentType: req.file.mimetype, + ContentType: detectedType.contentType, Metadata: { doc_id: docId }, })); - res.json({ s3Uri: `s3://${UPLOAD_BUCKET}/${s3Key}`, docId, filename, size: req.file.size }); + res.json({ + s3Uri: `s3://${UPLOAD_BUCKET}/${s3Key}`, + docId, + filename, + size: req.file.size, + contentType: detectedType.contentType, + fileKind: detectedType.fileKind, + }); } catch (e) { res.status(500).json({ error: e.message }); } diff --git a/ui/src/components/Chat.jsx b/ui/src/components/Chat.jsx index 50fde3f..18a0212 100644 --- a/ui/src/components/Chat.jsx +++ b/ui/src/components/Chat.jsx @@ -185,7 +185,7 @@ function MyComposer() { // ── S3 Upload Attachment Adapter ── class S3AttachmentAdapter { - accept = 'application/pdf' + accept = 'application/pdf,image/png,image/jpeg,image/tiff,.pdf,.png,.jpg,.jpeg,.tif,.tiff' // Top level of the job-tracking hierarchy (doc_id -> job_id -> subtask_id). // The server mints it per upload; we hold the most recent one so subsequent @@ -193,14 +193,24 @@ class S3AttachmentAdapter { lastDocId = '' async add({ file }) { - if (!file.name.toLowerCase().endsWith('.pdf') && file.type !== 'application/pdf') { - throw new Error('Only PDF files are supported') + const extension = file.name.toLowerCase().match(/\.[^.]+$/)?.[0] + const supportedTypes = { + '.pdf': 'application/pdf', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + } + const contentType = supportedTypes[extension] + if (!contentType) { + throw new Error('Supported formats: PDF, TIFF, TIF, JPEG, JPG, and PNG') } return { id: crypto.randomUUID(), - type: 'document', + type: contentType.startsWith('image/') ? 'image' : 'document', name: file.name, - contentType: 'application/pdf', + contentType, file, status: { type: 'requires-action', reason: 'composer-send' }, } @@ -217,13 +227,13 @@ class S3AttachmentAdapter { if (data.docId) this.lastDocId = data.docId - // Return the s3 URI as text content so the agent sees it + const uploadedKind = data.fileKind === 'image' ? 'image' : 'PDF' return { ...attachment, status: { type: 'complete' }, content: [{ type: 'text', - text: `Uploaded file: ${data.s3Uri}`, + text: `Uploaded ${uploadedKind} ready for analysis: ${data.s3Uri}`, }], } } @@ -258,10 +268,16 @@ function ChatInner() { const adapter = useMemo(() => ({ async *run({ messages, abortSignal }) { const lastUserMsg = [...messages].reverse().find(m => m.role === 'user') - const messageText = lastUserMsg?.content + const typedText = lastUserMsg?.content ?.filter(c => c.type === 'text') .map(c => c.text) .join('\n') || '' + const attachmentText = lastUserMsg?.attachments + ?.flatMap(attachment => attachment.content || []) + .filter(c => c.type === 'text') + .map(c => c.text) + .join('\n') || '' + const messageText = [typedText, attachmentText].filter(Boolean).join('\n') const res = await fetch('/api/chat', { method: 'POST', @@ -271,7 +287,7 @@ function ChatInner() { session_id: sessionId, audit_mode: auditMode, dynamic_tokens: dynamicTokens, - // Empty until a PDF has been uploaded in this session; the agent + // Empty until a document or image has been uploaded in this session; the agent // treats an absent doc_id as "not attributable to a document". doc_id: attachmentAdapter.lastDocId, }),