From 9682b308b829cbbded88a09690f27c379f4a8083 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 22 Nov 2026 17:01:23 +0700 Subject: [PATCH 1/2] fix(importer): harden review jobs against NaN cap and proto keys Closes #1853 --- .../shopee/__tests__/review-ai-points.test.ts | 256 +++++++++++++++++- .../shopee/__tests__/review-values.test.ts | 40 +++ .../src/shopee/jobs/review-ai-points.ts | 70 ++++- .../importer/src/shopee/jobs/review-values.ts | 15 +- apps/importer/src/shopee/logger.ts | 54 ++-- .../src/shopee/product-points-ai-prompt.ts | 4 +- apps/importer/src/shopee/review-ai-prompt.ts | 3 +- 7 files changed, 402 insertions(+), 40 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/review-ai-points.test.ts b/apps/importer/src/shopee/__tests__/review-ai-points.test.ts index 950205863..98a2773cc 100644 --- a/apps/importer/src/shopee/__tests__/review-ai-points.test.ts +++ b/apps/importer/src/shopee/__tests__/review-ai-points.test.ts @@ -1,6 +1,13 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { comparablePoint, storedPoint } from '@repo/shared-data/rules'; +import { + comparablePoint, + storedPoint, + type PointCluster, + type PointLine, +} from '@repo/shared-data/rules'; +import { sanitizeAiGeneratedText } from '../logger.js'; +import { mergeAndSelectAiProductPoints } from '../product-points-ai-prompt.js'; import { batchCostUsd, checkReviewPoints, @@ -20,9 +27,12 @@ import { collectBatch, dosAiReviewAiClient, fromDosAiOutputLine, + monthSpendUsd, parseReviewAiPointsArgs, qualityStop, runReviewAiPointsJob, + storeAiRows, + type AiBatchRow, type AiMarkerRow, type AiReviewRow, type AiStoreRow, @@ -1096,6 +1106,249 @@ function fakeSupabase(seed: Record) { return { client: client as unknown as RateSupabaseClient, tables, stores, writes, rpcs, state }; } +async function testHardening1853() { + const now = new Date('2026-11-20T00:00:00Z'); + const mkBatch = (extra: Partial = {}): AiBatchRow => ({ + id: 'batch_harden01', + model: 'dos', + prompt_version: REVIEW_AI_PROMPT_VERSION, + status: 'stored', + manifest: [], + reserved_usd: 0.01, + cost_usd: 1.25, + submitted_at: '2026-11-02T00:00:00Z', + stored_at: '2026-11-02T01:00:00Z', + ...extra, + }); + + // 1) monthSpendUsd & runReviewAiPointsJob: non-finite values fail closed (Infinity, never NaN), + // negative values cannot subtract from spend, and +00:00 month start is counted. + for (const bad of [ + mkBatch({ cost_usd: 'NaN', reserved_usd: 0.01 }), + mkBatch({ cost_usd: Number.NaN, reserved_usd: 0.01 }), + mkBatch({ cost_usd: null, reserved_usd: 'NaN' }), + mkBatch({ cost_usd: Number.POSITIVE_INFINITY, reserved_usd: 0.01 }), + mkBatch({ cost_usd: 'Infinity', reserved_usd: 0.01 }), + mkBatch({ cost_usd: '-Infinity', reserved_usd: 0.01 }), + mkBatch({ cost_usd: '', reserved_usd: 0.01 }), + ]) { + const spend = monthSpendUsd([bad, mkBatch({ id: 'batch_neg', cost_usd: -50 })], 'dos', now); + assert.equal(Number.isNaN(spend), false, 'monthSpendUsd must never be NaN'); + assert.equal(spend, Number.POSITIVE_INFINITY, 'malformed row fails closed to +Infinity'); + + const db = fakeSupabase({ + reviews: [review(1)], + review_points: [], + review_ai_extractions: [], + review_ai_batches: [bad], + products: listings, + master_products: masters, + categories, + }); + const ai = fakeAi(() => []); + const res = await runReviewAiPointsJob(db.client, { + apply: true, + enabled: true, + ai: ai.client, + now, + }); + assert.equal(res.capped, true, 'malformed batch row must cap the run'); + assert.equal(res.sent, 0); + assert.equal(ai.calls.length, 0, 'capped run makes zero model calls'); + } + + assert.equal( + monthSpendUsd( + [ + mkBatch({ id: 'b1', cost_usd: 4.5 }), + mkBatch({ id: 'b2', cost_usd: -10, reserved_usd: 0 }), + mkBatch({ id: 'b3', cost_usd: -2, reserved_usd: 0.4 }), + ], + 'dos', + now + ), + 4.9, + 'negative cost_usd falls back to positive reserved_usd or 0 and never subtracts' + ); + assert.equal( + monthSpendUsd( + [mkBatch({ submitted_at: '2026-11-01T00:00:00+00:00', cost_usd: 2.25 })], + 'dos', + now + ), + 2.25, + 'PostgREST +00:00 timestamp at month start is included' + ); + + // 2) batchCostUsd rejects Infinity / -Infinity / NaN / negative token counts. + assert.equal( + batchCostUsd('dos', { + input_tokens: Number.POSITIVE_INFINITY, + output_tokens: Number.NaN, + cache_read_input_tokens: Number.NEGATIVE_INFINITY, + cache_creation_input_tokens: -1000, + }), + 0 + ); + assert.equal( + batchCostUsd('dos', { + input_tokens: 1_000_000, + output_tokens: Number.POSITIVE_INFINITY, + }), + batchCostUsd('dos', { input_tokens: 1_000_000, output_tokens: 0 }) + ); + + // 3) Prototype-key guards in storeAiRows and mergeAndSelectAiProductPoints (byCriterion). + const protoRpcPayload = JSON.parse( + '{"__proto__":7,"constructor":9,"toString":11,"valueOf":3,"stored":4,"empty":"Infinity","rejected":-2}' + ) as Record; + const rpcClient = { + rpc: () => Promise.resolve({ data: protoRpcPayload, error: null }), + } as unknown as RateSupabaseClient; + const dummyRow: AiStoreRow = { + id: id(1), + text_hash: 'h1', + prompt_version: REVIEW_AI_PROMPT_VERSION, + criteria_version: 1, + category: 'smartphones', + model: 'dos', + batch_id: 'batch_01', + failed: false, + rejected: 0, + points: [], + }; + const counts = await storeAiRows(rpcClient, [dummyRow]); + assert.equal(Object.getPrototypeOf(counts), Object.prototype); + assert.equal(Object.hasOwn(counts, '__proto__'), false); + assert.equal(Object.hasOwn(counts, 'constructor'), false); + assert.equal(Object.hasOwn(counts, 'toString'), false); + assert.equal(Object.hasOwn(counts, 'valueOf'), false); + assert.equal(counts.toString, Object.prototype.toString); + assert.equal(counts.stored, 4); + assert.equal(counts.empty, 0); + assert.equal(counts.rejected, 0); + + const protoKeys = ['__proto__', 'constructor', 'toString', 'battery'] as const; + const pLines: PointLine[] = []; + const pClusters: PointCluster[] = []; + protoKeys.forEach((critKey, idx) => { + const l1 = `pl-${idx}-1`; + const l2 = `pl-${idx}-2`; + pLines.push( + { + id: l1, + reviewId: 'r1', + writerId: 'w1', + kind: 'pro', + criterionKey: critKey, + body: 'pin dùng bền cả ngày', + weight: 1, + isExpert: false, + writtenAt: now, + method: 'writer', + }, + { + id: l2, + reviewId: 'r2', + writerId: 'w2', + kind: 'pro', + criterionKey: critKey, + body: 'pin dùng bền cả ngày', + weight: 1, + isExpert: false, + writtenAt: now, + method: 'writer', + } + ); + pClusters.push({ + key: `cluster-${idx}`, + criterionKey: critKey, + kind: 'pro', + label: 'Pin dùng bền cả ngày', + support: 2, + reviewerCount: 2, + aiReviewerCount: 0, + hasExpert: false, + firstSeen: now, + lastSeen: now, + lineIds: [l1, l2], + visible: true, + }); + }); + const merged = mergeAndSelectAiProductPoints(pClusters, pLines, [], now); + assert.equal(Object.getPrototypeOf(merged.byCriterion), null); + assert.equal( + (merged.byCriterion as Record).valueOf, + undefined, + 'unpopulated Object.prototype keys stay undefined' + ); + assert.equal(({} as Record).pros, undefined, 'Object.prototype untouched'); + for (const critKey of protoKeys) { + assert.equal(merged.byCriterion[critKey]?.pros.length, 1, `own bucket kept for ${critKey}`); + } + assert.equal(merged.shown.length, 4); + + // 4) UTF-16 surrogate safety in dosAiReviewAiClient and sanitizeAiGeneratedText. + const seenPaths: string[] = []; + const surrogateGateway = (async (url: string | URL | Request) => { + const path = String(url).replace('https://api.dos.me/v1', ''); + seenPaths.push(path); + if (path.startsWith('/batches/') && path.endsWith('/cancel')) { + return new Response(JSON.stringify({ status: 'cancelling' }), { status: 200 }); + } + if (path.startsWith('/batches/')) { + return new Response( + JSON.stringify({ status: 'completed', output_file_id: 'file_\uDC00_out' }), + { status: 200 } + ); + } + if (path.startsWith('/files/') && path.endsWith('/content')) { + return new Response('', { status: 200 }); + } + return new Response('missing', { status: 404 }); + }) as typeof fetch; + const surrogateClient = dosAiReviewAiClient( + 'dos_sk_test', + undefined, + surrogateGateway, + async () => {} + ); + assert.equal(await surrogateClient.batchStatus('batch_\uD800_bad'), 'ended'); + const drained: BatchResult[] = []; + for await (const item of surrogateClient.results('batch_\uD800_bad')) drained.push(item); + assert.equal(drained.length, 0); + await surrogateClient.cancelBatch('batch_\uDFFF_bad'); + assert.deepEqual(seenPaths, [ + '/batches/batch_%EF%BF%BD_bad', + '/batches/batch_%EF%BF%BD_bad', + '/files/file_%EF%BF%BD_out/content', + '/batches/batch_%EF%BF%BD_bad/cancel', + ]); + + const cutPair = sanitizeAiGeneratedText('abcd😀', 5); + assert.equal(cutPair, 'abcd'); + assert.ok((cutPair as string & { isWellFormed(): boolean }).isWellFormed()); + assert.ok(cutPair.length <= 5); + + const cutEmojiAtOne = sanitizeAiGeneratedText('😀', 1); + assert.equal(cutEmojiAtOne, ''); + assert.ok((cutEmojiAtOne as string & { isWellFormed(): boolean }).isWellFormed()); + + const cutSpacePair = sanitizeAiGeneratedText('abcd 😀', 6); + assert.equal(cutSpacePair, 'abcd'); + assert.ok((cutSpacePair as string & { isWellFormed(): boolean }).isWellFormed()); + + const cutBoundedBoundary = sanitizeAiGeneratedText(`${'a'.repeat(4095)}😀`, 5000); + assert.ok((cutBoundedBoundary as string & { isWellFormed(): boolean }).isWellFormed()); + assert.equal(cutBoundedBoundary.includes('\uFFFD'), false); + + const loneInInput = sanitizeAiGeneratedText('abcd\uD800ef\uDC00', 5); + assert.ok((loneInInput as string & { isWellFormed(): boolean }).isWellFormed()); + assert.ok(loneInInput.length <= 5); + + console.log(' ✅ #1853 spend-cap NaN/Infinity, prototype keys, and UTF-16 surrogates verified'); +} + async function run() { console.log('Review AI points job'); testPrompt(); @@ -1103,6 +1356,7 @@ async function run() { testCollect(); await testJob(); await testDosAiClient(); + await testHardening1853(); } run().catch(err => { diff --git a/apps/importer/src/shopee/__tests__/review-values.test.ts b/apps/importer/src/shopee/__tests__/review-values.test.ts index 31258220c..17c466a3d 100644 --- a/apps/importer/src/shopee/__tests__/review-values.test.ts +++ b/apps/importer/src/shopee/__tests__/review-values.test.ts @@ -258,6 +258,46 @@ async function run() { assert.deepEqual(edge.get(M2)!.criteria_scores, { battery: { score: 4.133, weight_sum: 1.5, writers: 3 }, }); + // Hardening (#1853): NaN / Infinity in criteria_ratings, stars, or weights never reach BigInt, + // and prototype keys (__proto__, constructor, prototype) are ignored. + const protoCriteria = JSON.parse( + '{"__proto__":4,"constructor":4,"prototype":4,"battery":4}' + ) as Record; + const malformedRatings = expectedMasterRatings({ + reviews: [ + review(40, 'l2', { + rating_star: 5, + criteria_ratings: { + ...protoCriteria, + display: Number.NaN, + camera: Number.POSITIVE_INFINITY, + sound: Number.NEGATIVE_INFINITY, + }, + }), + review(41, 'l2', { + rating_star: 4, + criteria_ratings: { ...protoCriteria, display: Number.NaN }, + }), + review(42, 'l2', { + rating_star: Number.NaN, + criteria_ratings: { ...protoCriteria, display: Number.POSITIVE_INFINITY }, + }), + review(43, 'l2', { rating_exact: 'NaN', rating_star: 5 }), + review(44, 'l2', { rating_exact: Number.POSITIVE_INFINITY, rating_star: 5 }), + review(45, 'l2', { helpfulness_weight: Number.POSITIVE_INFINITY, rating_star: 5 }), + review(46, 'l2', { helpfulness_weight: 'NaN', rating_star: 5 }), + ], + listings, + values: new Map(), + }); + const m2Malformed = malformedRatings.get(M2)!; + assert.equal(Object.getPrototypeOf(m2Malformed.criteria_scores), Object.prototype); + assert.equal(Object.hasOwn(m2Malformed.criteria_scores, '__proto__'), false); + assert.equal(Object.hasOwn(m2Malformed.criteria_scores, 'constructor'), false); + assert.equal(Object.hasOwn(m2Malformed.criteria_scores, 'prototype'), false); + assert.deepEqual(Object.keys(m2Malformed.criteria_scores), ['battery']); + assert.equal(m2Malformed.criteria_scores.battery!.writers, 3); + assert.equal(m2Malformed.rating_weight_sum, 2, 'only finite 1..5 star reviews count in rating'); assert.ok( !masterRatingMatches( { diff --git a/apps/importer/src/shopee/jobs/review-ai-points.ts b/apps/importer/src/shopee/jobs/review-ai-points.ts index 8a0131804..ab87b4450 100644 --- a/apps/importer/src/shopee/jobs/review-ai-points.ts +++ b/apps/importer/src/shopee/jobs/review-ai-points.ts @@ -301,6 +301,8 @@ export function dosAiReviewAiClient( }; const json = async (path: string, init?: RequestInit, retry?: 'all' | 'throttled') => JSON.parse((await call(path, init, retry)).text) as T; + const encodePathId = (value: string): string => + encodeURIComponent((String(value) as string & { toWellFormed(): string }).toWellFormed()); return { async createBatch(requests) { const form = new FormData(); @@ -329,7 +331,7 @@ export function dosAiReviewAiClient( return { id: batch.id }; }, async batchStatus(id) { - const batch = await json<{ status: string }>(`/batches/${encodeURIComponent(id)}`); + const batch = await json<{ status: string }>(`/batches/${encodePathId(id)}`); if (batch.status === 'cancelling') return 'canceling'; return ['completed', 'failed', 'cancelled', 'expired'].includes(batch.status) ? 'ended' @@ -340,7 +342,7 @@ export function dosAiReviewAiClient( status?: string; output_file_id?: string | null; errmsg?: string | null; - }>(`/batches/${encodeURIComponent(id)}`); + }>(`/batches/${encodePathId(id)}`); if (!batch.output_file_id) { // Failed as a whole, or ended with no line run: nothing to store. throw new BatchGoneError( @@ -350,7 +352,7 @@ export function dosAiReviewAiClient( } let text: string; try { - text = (await call(`/files/${encodeURIComponent(batch.output_file_id)}/content`)).text; + text = (await call(`/files/${encodePathId(batch.output_file_id)}/content`)).text; } catch (err) { if (/ 404 /.test(String(err))) { throw new BatchGoneError(`dos.ai batch ${id}: its output file is gone (404)`, 'expired'); @@ -370,7 +372,7 @@ export function dosAiReviewAiClient( } }, async cancelBatch(id) { - await call(`/batches/${encodeURIComponent(id)}/cancel`, { method: 'POST' }); + await call(`/batches/${encodePathId(id)}/cancel`, { method: 'POST' }); }, }; } @@ -796,8 +798,13 @@ export async function storeAiRows( for (let attempt = 1; ; attempt++) { const { data, error } = await supabase.rpc('store_review_ai_points', { p_rows: part }); if (!error) { - for (const [key, n] of Object.entries((data ?? {}) as Record)) { - if (key in total) total[key as keyof StoreCounts] += Number(n) || 0; + if (data && typeof data === 'object' && !Array.isArray(data)) { + for (const [key, n] of Object.entries(data as Record)) { + if (Object.hasOwn(total, key)) { + const count = Number(n); + total[key as keyof StoreCounts] += Number.isFinite(count) && count > 0 ? count : 0; + } + } } break; } @@ -811,15 +818,43 @@ export async function storeAiRows( return total; } -const monthStart = (now: Date) => - new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)).toISOString(); +const monthStartMs = (now: Date) => Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1); + +function parseBatchUsd(raw: unknown): number | null { + if (raw === null || raw === undefined) return null; + if (typeof raw === 'number') { + return Number.isFinite(raw) ? raw : Number.POSITIVE_INFINITY; + } + if (typeof raw === 'string') { + const trimmed = raw.trim(); + if (!trimmed) return Number.POSITIVE_INFINITY; + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : Number.POSITIVE_INFINITY; + } + return Number.POSITIVE_INFINITY; +} /** This month's spend of a model: reported cost, or the reserved ceiling while open. */ export function monthSpendUsd(batches: readonly AiBatchRow[], model: string, now: Date): number { - const start = monthStart(now); - return batches - .filter(b => b.model === model && b.submitted_at >= start) - .reduce((s, b) => s + Number(b.cost_usd ?? b.reserved_usd ?? 0), 0); + const startMs = monthStartMs(now); + if (!Number.isFinite(startMs)) return Number.POSITIVE_INFINITY; + let total = 0; + for (const b of batches) { + if (b.model !== model) continue; + const submittedMs = + typeof b.submitted_at === 'string' ? Date.parse(b.submitted_at) : Number.NaN; + if (!Number.isFinite(submittedMs)) return Number.POSITIVE_INFINITY; + if (submittedMs < startMs) continue; + const cost = parseBatchUsd(b.cost_usd); + const reserved = parseBatchUsd(b.reserved_usd); + if (cost === Number.POSITIVE_INFINITY || reserved === Number.POSITIVE_INFINITY) { + return Number.POSITIVE_INFINITY; + } + const safeReserved = reserved !== null && reserved > 0 ? reserved : 0; + const rowUsd = cost !== null ? (cost >= 0 ? cost : safeReserved) : safeReserved; + total += rowUsd; + } + return total; } const BATCH_COLUMNS = @@ -1121,10 +1156,17 @@ export async function runReviewAiPointsJob( ); result.monthSpendUsd = monthSpendUsd(refreshed, model, now); const cap = REVIEW_AI_MODELS[model].monthlyCapUsd; - result.capped = requests.length > 0 && result.monthSpendUsd + result.reservedUsd > cap; + result.capped = + requests.length > 0 && + (!Number.isFinite(result.monthSpendUsd) || + !Number.isFinite(result.reservedUsd) || + result.monthSpendUsd + result.reservedUsd > cap); + const monthText = Number.isFinite(result.monthSpendUsd) + ? `$${result.monthSpendUsd.toFixed(4)}` + : 'non-finite (fail-closed)'; logger.info( 'review_ai.plan', - `${apply ? 'Sending' : 'Would send'} ${chosen.length} of ${candidates.length} due reviews in ${requests.length} requests; ceiling $${result.reservedUsd.toFixed(4)}, month $${result.monthSpendUsd.toFixed(4)} of $${cap}.`, + `${apply ? 'Sending' : 'Would send'} ${chosen.length} of ${candidates.length} due reviews in ${requests.length} requests; ceiling $${result.reservedUsd.toFixed(4)}, month ${monthText} of $${cap}.`, { apply, model, diff --git a/apps/importer/src/shopee/jobs/review-values.ts b/apps/importer/src/shopee/jobs/review-values.ts index c6e5e3e31..4b0025ae2 100644 --- a/apps/importer/src/shopee/jobs/review-values.ts +++ b/apps/importer/src/shopee/jobs/review-values.ts @@ -346,6 +346,7 @@ export interface MasterRatingExpected { } const CRITERION_KEY = /^[a-z_]{2,32}$/; +const DISALLOWED_CRITERION_KEYS = new Set(['__proto__', 'constructor', 'prototype']); /** * P2's rating with P6's weight, per master (cluster_id scope): user reviews @@ -374,7 +375,8 @@ export function expectedMasterRatings(input: { review.reviewer_type !== 'user' || flag === null || flag.include_in_rating === false || - !(weight > 0) + !Number.isFinite(weight) || + weight <= 0 ) { continue; } @@ -392,14 +394,21 @@ export function expectedMasterRatings(input: { } const out = new Map(); for (const [masterId, eligible] of byMaster) { - const rated = eligible.filter(e => e.star >= 1 && e.star <= 5); + const rated = eligible.filter(e => Number.isFinite(e.star) && e.star >= 1 && e.star <= 5); const sumW = rated.reduce((s, e) => s + e.w, 0n); const sumWS = rated.reduce((s, e) => s + e.w * toUnits(e.star, V_UNIT), 0n); const perKey = new Map(); for (const e of eligible) { if (!e.criteria || typeof e.criteria !== 'object' || Array.isArray(e.criteria)) continue; for (const [key, value] of Object.entries(e.criteria as Record)) { - if (!CRITERION_KEY.test(key) || typeof value !== 'number' || value < 1 || value > 5) { + if ( + !CRITERION_KEY.test(key) || + DISALLOWED_CRITERION_KEYS.has(key) || + typeof value !== 'number' || + !Number.isFinite(value) || + value < 1 || + value > 5 + ) { continue; } const acc = perKey.get(key) ?? { writers: 0, w: 0n, shown: 0n, ws: 0n }; diff --git a/apps/importer/src/shopee/logger.ts b/apps/importer/src/shopee/logger.ts index 538ab5cac..e3524522e 100644 --- a/apps/importer/src/shopee/logger.ts +++ b/apps/importer/src/shopee/logger.ts @@ -129,6 +129,22 @@ export function computeRetryDelayMs( return Math.min(safeMaxDelayMs, half + jitter); } +const toWellFormedUtf16 = (value: string): string => + (value as string & { toWellFormed(): string }).toWellFormed(); + +function sliceUtf16WellFormed(value: string, maxUnits: number): string { + if (value.length <= maxUnits) return toWellFormedUtf16(value); + let end = maxUnits; + if (end > 0) { + const prev = value.charCodeAt(end - 1); + const next = value.charCodeAt(end); + if (prev >= 0xd800 && prev <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) { + end -= 1; + } + } + return toWellFormedUtf16(value.slice(0, end)); +} + /** * Sanitizes untrusted AI-generated output text (`ai_summary`, `pros`, `cons`, `reason`) * by stripping HTML tags, control characters, URLs, and phone/Zalo numbers to prevent @@ -140,26 +156,24 @@ export function sanitizeAiGeneratedText(raw: unknown, maxLen = 400): string { typeof maxLen === 'number' && Number.isFinite(maxLen) && maxLen >= 1 ? Math.min(Math.floor(maxLen), MAX_LOG_STRING_LENGTH) : 400; - const boundedInput = raw.slice(0, Math.max(safeMaxLen * 4, 4096)); - return ( - boundedInput - // `[^<>]` rather than `[^>]`: a run of unclosed '<' would otherwise be - // rescanned to the end of the string from each '<' (quadratic). - .replace(/<[^<>]*>/g, ' ') - // Strip explicit http(s):// and www. URLs - .replace(/\b(?:https?:\/\/|www\.)\S+/gi, '[link-removed]') - // Strip bare domain links with common TLDs/paths (e.g. bit.ly/xyz, shopee.vn/abc, t.me/xyz) - .replace( - /\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.(?:vn|com|net|org|io|me|ly|gg|co|shop|store|xyz|info)(?:\/\S*)?\b/gi, - '[link-removed]' - ) - // Strip Vietnamese phone / Zalo numbers (e.g. 0912345678, +84 912 345 678, 098.123.4567) - .replace(/(?:\+?84|0)\s*[1-9](?:[\s.-]*\d){7,9}\b/g, '[phone-removed]') - .replace(/\p{Cc}+/gu, ' ') - .replace(/\s+/g, ' ') - .trim() - .slice(0, safeMaxLen) - ); + const boundedInput = sliceUtf16WellFormed(raw, Math.max(safeMaxLen * 4, 4096)); + const cleaned = boundedInput + // `[^<>]` rather than `[^>]`: a run of unclosed '<' would otherwise be + // rescanned to the end of the string from each '<' (quadratic). + .replace(/<[^<>]*>/g, ' ') + // Strip explicit http(s):// and www. URLs + .replace(/\b(?:https?:\/\/|www\.)\S+/gi, '[link-removed]') + // Strip bare domain links with common TLDs/paths (e.g. bit.ly/xyz, shopee.vn/abc, t.me/xyz) + .replace( + /\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.(?:vn|com|net|org|io|me|ly|gg|co|shop|store|xyz|info)(?:\/\S*)?\b/gi, + '[link-removed]' + ) + // Strip Vietnamese phone / Zalo numbers (e.g. 0912345678, +84 912 345 678, 098.123.4567) + .replace(/(?:\+?84|0)\s*[1-9](?:[\s.-]*\d){7,9}\b/g, '[phone-removed]') + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); + return sliceUtf16WellFormed(cleaned, safeMaxLen).trimEnd(); } /** diff --git a/apps/importer/src/shopee/product-points-ai-prompt.ts b/apps/importer/src/shopee/product-points-ai-prompt.ts index bd7a8a09f..ad67abd19 100644 --- a/apps/importer/src/shopee/product-points-ai-prompt.ts +++ b/apps/importer/src/shopee/product-points-ai-prompt.ts @@ -324,7 +324,9 @@ export function mergeAndSelectAiProductPoints( pros: list.filter(c => c.kind === 'pro').slice(0, max), cons: list.filter(c => c.kind === 'con').slice(0, max), }); - const byCriterion: Record = {}; + const byCriterion: Record = Object.create( + null + ) as Record; for (const key of new Set(rebuilt.map(c => c.criterionKey ?? GENERAL_POINTS))) { byCriterion[key] = pickSides( rebuilt.filter(c => (c.criterionKey ?? GENERAL_POINTS) === key), diff --git a/apps/importer/src/shopee/review-ai-prompt.ts b/apps/importer/src/shopee/review-ai-prompt.ts index 36991a646..14af1d833 100644 --- a/apps/importer/src/shopee/review-ai-prompt.ts +++ b/apps/importer/src/shopee/review-ai-prompt.ts @@ -367,7 +367,8 @@ export function batchCostUsd( } ): number { const price = REVIEW_AI_MODELS[model]; - const n = (v: number | null | undefined) => (typeof v === 'number' && v > 0 ? v : 0); + const n = (v: number | null | undefined) => + typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0; const usd = (n(usage.input_tokens) * price.input + n(usage.cache_read_input_tokens) * price.input * CACHE_READ_FACTOR + From 107f6fc7cba77c74cbf680bdc543f17784be6ddd Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sun, 22 Nov 2026 17:16:53 +0700 Subject: [PATCH 2/2] fix(importer): close review-gate path, overflow, and weight bounds --- .../shopee/__tests__/review-ai-points.test.ts | 36 +++++++++++-- .../shopee/__tests__/review-values.test.ts | 7 ++- .../src/shopee/jobs/review-ai-points.ts | 50 +++++++++++++++---- .../importer/src/shopee/jobs/review-values.ts | 2 +- apps/importer/src/shopee/review-ai-prompt.ts | 17 ++++--- 5 files changed, 88 insertions(+), 24 deletions(-) diff --git a/apps/importer/src/shopee/__tests__/review-ai-points.test.ts b/apps/importer/src/shopee/__tests__/review-ai-points.test.ts index 98a2773cc..0f4d6d8e3 100644 --- a/apps/importer/src/shopee/__tests__/review-ai-points.test.ts +++ b/apps/importer/src/shopee/__tests__/review-ai-points.test.ts @@ -1127,6 +1127,8 @@ async function testHardening1853() { mkBatch({ cost_usd: 'NaN', reserved_usd: 0.01 }), mkBatch({ cost_usd: Number.NaN, reserved_usd: 0.01 }), mkBatch({ cost_usd: null, reserved_usd: 'NaN' }), + mkBatch({ cost_usd: null, reserved_usd: null as unknown as number }), + mkBatch({ cost_usd: null, reserved_usd: -1 }), mkBatch({ cost_usd: Number.POSITIVE_INFINITY, reserved_usd: 0.01 }), mkBatch({ cost_usd: 'Infinity', reserved_usd: 0.01 }), mkBatch({ cost_usd: '-Infinity', reserved_usd: 0.01 }), @@ -1172,15 +1174,16 @@ async function testHardening1853() { ); assert.equal( monthSpendUsd( - [mkBatch({ submitted_at: '2026-11-01T00:00:00+00:00', cost_usd: 2.25 })], + [mkBatch({ model: 'dos ', submitted_at: '2026-11-01T00:00:00+00:00', cost_usd: 2.25 })], 'dos', now ), 2.25, - 'PostgREST +00:00 timestamp at month start is included' + 'PostgREST +00:00 timestamp at month start and padded model are included' ); - // 2) batchCostUsd rejects Infinity / -Infinity / NaN / negative token counts. + // 2) batchCostUsd rejects Infinity / -Infinity / NaN / negative token counts, prototype model + // keys, and does not overflow to Infinity or $0 on huge finite token counts. assert.equal( batchCostUsd('dos', { input_tokens: Number.POSITIVE_INFINITY, @@ -1190,6 +1193,26 @@ async function testHardening1853() { }), 0 ); + assert.equal(batchCostUsd('toString' as never, { input_tokens: 1000 }), 0); + const maxValCost = batchCostUsd('qwen3.8-max', { output_tokens: Number.MAX_VALUE }); + assert.ok(Number.isFinite(maxValCost) && maxValCost > 0); + const hugeCollected = collectBatch( + { + id: 'batch_huge', + model: 'dos', + prompt_version: REVIEW_AI_PROMPT_VERSION, + manifest: [ + { custom_id: 'r1', category: 'smartphones', criteria_version: 1, reviews: [] }, + { custom_id: 'r2', category: 'smartphones', criteria_version: 1, reviews: [] }, + ], + }, + [ + { custom_id: 'r1', ...answer([], 'end_turn', { input_tokens: 1e308, output_tokens: 1e308 }) }, + { custom_id: 'r2', ...answer([], 'end_turn', { input_tokens: 1e308, output_tokens: 1e308 }) }, + ], + new Map() + ); + assert.ok(Number.isFinite(hugeCollected.costUsd) && hugeCollected.costUsd > 0); assert.equal( batchCostUsd('dos', { input_tokens: 1_000_000, @@ -1324,6 +1347,10 @@ async function testHardening1853() { '/files/file_%EF%BF%BD_out/content', '/batches/batch_%EF%BF%BD_bad/cancel', ]); + for (const badId of ['', ' ', '.', '..', 'a/b', 'a?b', 'a#b', 'a\\b', 'a\u0000b']) { + await assert.rejects(surrogateClient.batchStatus(badId), /Invalid dos\.ai resource identifier/); + await assert.rejects(surrogateClient.cancelBatch(badId), /Invalid dos\.ai resource identifier/); + } const cutPair = sanitizeAiGeneratedText('abcd😀', 5); assert.equal(cutPair, 'abcd'); @@ -1338,7 +1365,8 @@ async function testHardening1853() { assert.equal(cutSpacePair, 'abcd'); assert.ok((cutSpacePair as string & { isWellFormed(): boolean }).isWellFormed()); - const cutBoundedBoundary = sanitizeAiGeneratedText(`${'a'.repeat(4095)}😀`, 5000); + const cutBoundedBoundary = sanitizeAiGeneratedText(`a${' '.repeat(4094)}😀`, 400); + assert.equal(cutBoundedBoundary, 'a'); assert.ok((cutBoundedBoundary as string & { isWellFormed(): boolean }).isWellFormed()); assert.equal(cutBoundedBoundary.includes('\uFFFD'), false); diff --git a/apps/importer/src/shopee/__tests__/review-values.test.ts b/apps/importer/src/shopee/__tests__/review-values.test.ts index 17c466a3d..9f08c8273 100644 --- a/apps/importer/src/shopee/__tests__/review-values.test.ts +++ b/apps/importer/src/shopee/__tests__/review-values.test.ts @@ -286,6 +286,7 @@ async function run() { review(44, 'l2', { rating_exact: Number.POSITIVE_INFINITY, rating_star: 5 }), review(45, 'l2', { helpfulness_weight: Number.POSITIVE_INFINITY, rating_star: 5 }), review(46, 'l2', { helpfulness_weight: 'NaN', rating_star: 5 }), + review(47, 'l2', { helpfulness_weight: Number.MAX_VALUE, rating_star: 5 }), ], listings, values: new Map(), @@ -297,7 +298,11 @@ async function run() { assert.equal(Object.hasOwn(m2Malformed.criteria_scores, 'prototype'), false); assert.deepEqual(Object.keys(m2Malformed.criteria_scores), ['battery']); assert.equal(m2Malformed.criteria_scores.battery!.writers, 3); - assert.equal(m2Malformed.rating_weight_sum, 2, 'only finite 1..5 star reviews count in rating'); + assert.equal( + m2Malformed.rating_weight_sum, + 5, + 'only finite 1..5 star reviews count and huge finite weight clamps cleanly to 3' + ); assert.ok( !masterRatingMatches( { diff --git a/apps/importer/src/shopee/jobs/review-ai-points.ts b/apps/importer/src/shopee/jobs/review-ai-points.ts index ab87b4450..f8dc2bc51 100644 --- a/apps/importer/src/shopee/jobs/review-ai-points.ts +++ b/apps/importer/src/shopee/jobs/review-ai-points.ts @@ -301,8 +301,21 @@ export function dosAiReviewAiClient( }; const json = async (path: string, init?: RequestInit, retry?: 'all' | 'throttled') => JSON.parse((await call(path, init, retry)).text) as T; - const encodePathId = (value: string): string => - encodeURIComponent((String(value) as string & { toWellFormed(): string }).toWellFormed()); + const normalizePathId = (value: unknown): string => { + if (typeof value !== 'string') throw new Error('Invalid dos.ai resource identifier.'); + const wellFormed = (value as string & { toWellFormed(): string }).toWellFormed().trim(); + if ( + !wellFormed || + wellFormed === '.' || + wellFormed === '..' || + wellFormed.length > 256 || + /[\s/?#\\\p{Cc}]/u.test(wellFormed) + ) { + throw new Error('Invalid dos.ai resource identifier.'); + } + return wellFormed; + }; + const encodePathId = (value: unknown): string => encodeURIComponent(normalizePathId(value)); return { async createBatch(requests) { const form = new FormData(); @@ -315,20 +328,21 @@ export function dosAiReviewAiClient( 'review-ai-points.jsonl' ); const file = await json<{ id: string }>('/files', { method: 'POST', body: form }); + const fileId = normalizePathId(file?.id); const batch = await json<{ id: string }>( '/batches', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - input_file_id: file.id, + input_file_id: fileId, endpoint: '/v1/chat/completions', completion_window: '24h', }), }, 'throttled' ); - return { id: batch.id }; + return { id: normalizePathId(batch?.id) }; }, async batchStatus(id) { const batch = await json<{ status: string }>(`/batches/${encodePathId(id)}`); @@ -702,7 +716,12 @@ export function collectBatch( const message = result.result.message; for (const key of Object.keys(usage) as (keyof typeof usage)[]) { const n = Number(message.usage?.[key] ?? 0); - usage[key] += Number.isFinite(n) && n > 0 ? n : 0; + if (Number.isFinite(n) && n > 0) { + const next = usage[key] + n; + usage[key] = Number.isFinite(next) + ? Math.min(next, Number.MAX_SAFE_INTEGER) + : Number.MAX_SAFE_INTEGER; + } } if (message.stop_reason === 'end_turn') { const parsed = parseOutput(message) as { reviews?: unknown } | null; @@ -837,10 +856,13 @@ function parseBatchUsd(raw: unknown): number | null { /** This month's spend of a model: reported cost, or the reserved ceiling while open. */ export function monthSpendUsd(batches: readonly AiBatchRow[], model: string, now: Date): number { const startMs = monthStartMs(now); - if (!Number.isFinite(startMs)) return Number.POSITIVE_INFINITY; + if (!Number.isFinite(startMs) || typeof model !== 'string') return Number.POSITIVE_INFINITY; + const targetModel = model.trim(); let total = 0; for (const b of batches) { - if (b.model !== model) continue; + if (!b || typeof b !== 'object') return Number.POSITIVE_INFINITY; + const rowModel = typeof b.model === 'string' ? b.model.trim() : ''; + if (rowModel !== targetModel) continue; const submittedMs = typeof b.submitted_at === 'string' ? Date.parse(b.submitted_at) : Number.NaN; if (!Number.isFinite(submittedMs)) return Number.POSITIVE_INFINITY; @@ -850,9 +872,17 @@ export function monthSpendUsd(batches: readonly AiBatchRow[], model: string, now if (cost === Number.POSITIVE_INFINITY || reserved === Number.POSITIVE_INFINITY) { return Number.POSITIVE_INFINITY; } - const safeReserved = reserved !== null && reserved > 0 ? reserved : 0; - const rowUsd = cost !== null ? (cost >= 0 ? cost : safeReserved) : safeReserved; - total += rowUsd; + if (cost === null) { + if (reserved === null || reserved < 0) return Number.POSITIVE_INFINITY; + total += reserved; + continue; + } + if (cost < 0) { + if (reserved === null || reserved < 0) return Number.POSITIVE_INFINITY; + total += reserved; + continue; + } + total += cost; } return total; } diff --git a/apps/importer/src/shopee/jobs/review-values.ts b/apps/importer/src/shopee/jobs/review-values.ts index 4b0025ae2..18caff945 100644 --- a/apps/importer/src/shopee/jobs/review-values.ts +++ b/apps/importer/src/shopee/jobs/review-values.ts @@ -383,7 +383,7 @@ export function expectedMasterRatings(input: { const exact = review.rating_exact == null ? null : Number(review.rating_exact); const list = byMaster.get(masterId) ?? []; const factor = completenessWeightFactor(input.values.get(review.id)?.weightCompleteness); - const w = toUnits(weight, 100n) * toUnits(factor, V_UNIT); + const w = toUnits(Math.min(weight, 3), 100n) * toUnits(factor, V_UNIT); list.push({ // LEAST(3, helpfulness_weight x factor), as reviewRatingWeight, in millionths. w: w < 3n * W_UNIT ? w : 3n * W_UNIT, diff --git a/apps/importer/src/shopee/review-ai-prompt.ts b/apps/importer/src/shopee/review-ai-prompt.ts index 14af1d833..84bc0d271 100644 --- a/apps/importer/src/shopee/review-ai-prompt.ts +++ b/apps/importer/src/shopee/review-ai-prompt.ts @@ -366,16 +366,17 @@ export function batchCostUsd( cache_creation_input_tokens?: number | null; } ): number { + if (!isReviewAiModel(model)) return 0; const price = REVIEW_AI_MODELS[model]; - const n = (v: number | null | undefined) => - typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0; + const mTokens = (v: number | null | undefined) => + typeof v === 'number' && Number.isFinite(v) && v > 0 ? v / 1_000_000 : 0; const usd = - (n(usage.input_tokens) * price.input + - n(usage.cache_read_input_tokens) * price.input * CACHE_READ_FACTOR + - n(usage.cache_creation_input_tokens) * price.input * CACHE_WRITE_FACTOR + - n(usage.output_tokens) * price.output) / - 1_000_000; - return usd * BATCH_DISCOUNT; + (mTokens(usage.input_tokens) * price.input + + mTokens(usage.cache_read_input_tokens) * price.input * CACHE_READ_FACTOR + + mTokens(usage.cache_creation_input_tokens) * price.input * CACHE_WRITE_FACTOR + + mTokens(usage.output_tokens) * price.output) * + BATCH_DISCOUNT; + return Number.isFinite(usd) ? usd : Number.POSITIVE_INFINITY; } /**