diff --git a/apps/importer/src/shopee/__tests__/review-ai-points.test.ts b/apps/importer/src/shopee/__tests__/review-ai-points.test.ts index 95020586..0f4d6d8e 100644 --- a/apps/importer/src/shopee/__tests__/review-ai-points.test.ts +++ b/apps/importer/src/shopee/__tests__/review-ai-points.test.ts @@ -1,6 +1,13 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { comparablePoint, storedPoint } from '@repo/shared-data/rules'; +import { + comparablePoint, + storedPoint, + type PointCluster, + type PointLine, +} from '@repo/shared-data/rules'; +import { sanitizeAiGeneratedText } from '../logger.js'; +import { mergeAndSelectAiProductPoints } from '../product-points-ai-prompt.js'; import { batchCostUsd, checkReviewPoints, @@ -20,9 +27,12 @@ import { collectBatch, dosAiReviewAiClient, fromDosAiOutputLine, + monthSpendUsd, parseReviewAiPointsArgs, qualityStop, runReviewAiPointsJob, + storeAiRows, + type AiBatchRow, type AiMarkerRow, type AiReviewRow, type AiStoreRow, @@ -1096,6 +1106,277 @@ function fakeSupabase(seed: Record) { return { client: client as unknown as RateSupabaseClient, tables, stores, writes, rpcs, state }; } +async function testHardening1853() { + const now = new Date('2026-11-20T00:00:00Z'); + const mkBatch = (extra: Partial = {}): AiBatchRow => ({ + id: 'batch_harden01', + model: 'dos', + prompt_version: REVIEW_AI_PROMPT_VERSION, + status: 'stored', + manifest: [], + reserved_usd: 0.01, + cost_usd: 1.25, + submitted_at: '2026-11-02T00:00:00Z', + stored_at: '2026-11-02T01:00:00Z', + ...extra, + }); + + // 1) monthSpendUsd & runReviewAiPointsJob: non-finite values fail closed (Infinity, never NaN), + // negative values cannot subtract from spend, and +00:00 month start is counted. + for (const bad of [ + mkBatch({ cost_usd: 'NaN', reserved_usd: 0.01 }), + mkBatch({ cost_usd: Number.NaN, reserved_usd: 0.01 }), + mkBatch({ cost_usd: null, reserved_usd: 'NaN' }), + mkBatch({ cost_usd: null, reserved_usd: null as unknown as number }), + mkBatch({ cost_usd: null, reserved_usd: -1 }), + mkBatch({ cost_usd: Number.POSITIVE_INFINITY, reserved_usd: 0.01 }), + mkBatch({ cost_usd: 'Infinity', reserved_usd: 0.01 }), + mkBatch({ cost_usd: '-Infinity', reserved_usd: 0.01 }), + mkBatch({ cost_usd: '', reserved_usd: 0.01 }), + ]) { + const spend = monthSpendUsd([bad, mkBatch({ id: 'batch_neg', cost_usd: -50 })], 'dos', now); + assert.equal(Number.isNaN(spend), false, 'monthSpendUsd must never be NaN'); + assert.equal(spend, Number.POSITIVE_INFINITY, 'malformed row fails closed to +Infinity'); + + const db = fakeSupabase({ + reviews: [review(1)], + review_points: [], + review_ai_extractions: [], + review_ai_batches: [bad], + products: listings, + master_products: masters, + categories, + }); + const ai = fakeAi(() => []); + const res = await runReviewAiPointsJob(db.client, { + apply: true, + enabled: true, + ai: ai.client, + now, + }); + assert.equal(res.capped, true, 'malformed batch row must cap the run'); + assert.equal(res.sent, 0); + assert.equal(ai.calls.length, 0, 'capped run makes zero model calls'); + } + + assert.equal( + monthSpendUsd( + [ + mkBatch({ id: 'b1', cost_usd: 4.5 }), + mkBatch({ id: 'b2', cost_usd: -10, reserved_usd: 0 }), + mkBatch({ id: 'b3', cost_usd: -2, reserved_usd: 0.4 }), + ], + 'dos', + now + ), + 4.9, + 'negative cost_usd falls back to positive reserved_usd or 0 and never subtracts' + ); + assert.equal( + monthSpendUsd( + [mkBatch({ model: 'dos ', submitted_at: '2026-11-01T00:00:00+00:00', cost_usd: 2.25 })], + 'dos', + now + ), + 2.25, + 'PostgREST +00:00 timestamp at month start and padded model are included' + ); + + // 2) batchCostUsd rejects Infinity / -Infinity / NaN / negative token counts, prototype model + // keys, and does not overflow to Infinity or $0 on huge finite token counts. + assert.equal( + batchCostUsd('dos', { + input_tokens: Number.POSITIVE_INFINITY, + output_tokens: Number.NaN, + cache_read_input_tokens: Number.NEGATIVE_INFINITY, + cache_creation_input_tokens: -1000, + }), + 0 + ); + assert.equal(batchCostUsd('toString' as never, { input_tokens: 1000 }), 0); + const maxValCost = batchCostUsd('qwen3.8-max', { output_tokens: Number.MAX_VALUE }); + assert.ok(Number.isFinite(maxValCost) && maxValCost > 0); + const hugeCollected = collectBatch( + { + id: 'batch_huge', + model: 'dos', + prompt_version: REVIEW_AI_PROMPT_VERSION, + manifest: [ + { custom_id: 'r1', category: 'smartphones', criteria_version: 1, reviews: [] }, + { custom_id: 'r2', category: 'smartphones', criteria_version: 1, reviews: [] }, + ], + }, + [ + { custom_id: 'r1', ...answer([], 'end_turn', { input_tokens: 1e308, output_tokens: 1e308 }) }, + { custom_id: 'r2', ...answer([], 'end_turn', { input_tokens: 1e308, output_tokens: 1e308 }) }, + ], + new Map() + ); + assert.ok(Number.isFinite(hugeCollected.costUsd) && hugeCollected.costUsd > 0); + assert.equal( + batchCostUsd('dos', { + input_tokens: 1_000_000, + output_tokens: Number.POSITIVE_INFINITY, + }), + batchCostUsd('dos', { input_tokens: 1_000_000, output_tokens: 0 }) + ); + + // 3) Prototype-key guards in storeAiRows and mergeAndSelectAiProductPoints (byCriterion). + const protoRpcPayload = JSON.parse( + '{"__proto__":7,"constructor":9,"toString":11,"valueOf":3,"stored":4,"empty":"Infinity","rejected":-2}' + ) as Record; + const rpcClient = { + rpc: () => Promise.resolve({ data: protoRpcPayload, error: null }), + } as unknown as RateSupabaseClient; + const dummyRow: AiStoreRow = { + id: id(1), + text_hash: 'h1', + prompt_version: REVIEW_AI_PROMPT_VERSION, + criteria_version: 1, + category: 'smartphones', + model: 'dos', + batch_id: 'batch_01', + failed: false, + rejected: 0, + points: [], + }; + const counts = await storeAiRows(rpcClient, [dummyRow]); + assert.equal(Object.getPrototypeOf(counts), Object.prototype); + assert.equal(Object.hasOwn(counts, '__proto__'), false); + assert.equal(Object.hasOwn(counts, 'constructor'), false); + assert.equal(Object.hasOwn(counts, 'toString'), false); + assert.equal(Object.hasOwn(counts, 'valueOf'), false); + assert.equal(counts.toString, Object.prototype.toString); + assert.equal(counts.stored, 4); + assert.equal(counts.empty, 0); + assert.equal(counts.rejected, 0); + + const protoKeys = ['__proto__', 'constructor', 'toString', 'battery'] as const; + const pLines: PointLine[] = []; + const pClusters: PointCluster[] = []; + protoKeys.forEach((critKey, idx) => { + const l1 = `pl-${idx}-1`; + const l2 = `pl-${idx}-2`; + pLines.push( + { + id: l1, + reviewId: 'r1', + writerId: 'w1', + kind: 'pro', + criterionKey: critKey, + body: 'pin dùng bền cả ngày', + weight: 1, + isExpert: false, + writtenAt: now, + method: 'writer', + }, + { + id: l2, + reviewId: 'r2', + writerId: 'w2', + kind: 'pro', + criterionKey: critKey, + body: 'pin dùng bền cả ngày', + weight: 1, + isExpert: false, + writtenAt: now, + method: 'writer', + } + ); + pClusters.push({ + key: `cluster-${idx}`, + criterionKey: critKey, + kind: 'pro', + label: 'Pin dùng bền cả ngày', + support: 2, + reviewerCount: 2, + aiReviewerCount: 0, + hasExpert: false, + firstSeen: now, + lastSeen: now, + lineIds: [l1, l2], + visible: true, + }); + }); + const merged = mergeAndSelectAiProductPoints(pClusters, pLines, [], now); + assert.equal(Object.getPrototypeOf(merged.byCriterion), null); + assert.equal( + (merged.byCriterion as Record).valueOf, + undefined, + 'unpopulated Object.prototype keys stay undefined' + ); + assert.equal(({} as Record).pros, undefined, 'Object.prototype untouched'); + for (const critKey of protoKeys) { + assert.equal(merged.byCriterion[critKey]?.pros.length, 1, `own bucket kept for ${critKey}`); + } + assert.equal(merged.shown.length, 4); + + // 4) UTF-16 surrogate safety in dosAiReviewAiClient and sanitizeAiGeneratedText. + const seenPaths: string[] = []; + const surrogateGateway = (async (url: string | URL | Request) => { + const path = String(url).replace('https://api.dos.me/v1', ''); + seenPaths.push(path); + if (path.startsWith('/batches/') && path.endsWith('/cancel')) { + return new Response(JSON.stringify({ status: 'cancelling' }), { status: 200 }); + } + if (path.startsWith('/batches/')) { + return new Response( + JSON.stringify({ status: 'completed', output_file_id: 'file_\uDC00_out' }), + { status: 200 } + ); + } + if (path.startsWith('/files/') && path.endsWith('/content')) { + return new Response('', { status: 200 }); + } + return new Response('missing', { status: 404 }); + }) as typeof fetch; + const surrogateClient = dosAiReviewAiClient( + 'dos_sk_test', + undefined, + surrogateGateway, + async () => {} + ); + assert.equal(await surrogateClient.batchStatus('batch_\uD800_bad'), 'ended'); + const drained: BatchResult[] = []; + for await (const item of surrogateClient.results('batch_\uD800_bad')) drained.push(item); + assert.equal(drained.length, 0); + await surrogateClient.cancelBatch('batch_\uDFFF_bad'); + assert.deepEqual(seenPaths, [ + '/batches/batch_%EF%BF%BD_bad', + '/batches/batch_%EF%BF%BD_bad', + '/files/file_%EF%BF%BD_out/content', + '/batches/batch_%EF%BF%BD_bad/cancel', + ]); + for (const badId of ['', ' ', '.', '..', 'a/b', 'a?b', 'a#b', 'a\\b', 'a\u0000b']) { + await assert.rejects(surrogateClient.batchStatus(badId), /Invalid dos\.ai resource identifier/); + await assert.rejects(surrogateClient.cancelBatch(badId), /Invalid dos\.ai resource identifier/); + } + + const cutPair = sanitizeAiGeneratedText('abcd😀', 5); + assert.equal(cutPair, 'abcd'); + assert.ok((cutPair as string & { isWellFormed(): boolean }).isWellFormed()); + assert.ok(cutPair.length <= 5); + + const cutEmojiAtOne = sanitizeAiGeneratedText('😀', 1); + assert.equal(cutEmojiAtOne, ''); + assert.ok((cutEmojiAtOne as string & { isWellFormed(): boolean }).isWellFormed()); + + const cutSpacePair = sanitizeAiGeneratedText('abcd 😀', 6); + assert.equal(cutSpacePair, 'abcd'); + assert.ok((cutSpacePair as string & { isWellFormed(): boolean }).isWellFormed()); + + const cutBoundedBoundary = sanitizeAiGeneratedText(`a${' '.repeat(4094)}😀`, 400); + assert.equal(cutBoundedBoundary, 'a'); + assert.ok((cutBoundedBoundary as string & { isWellFormed(): boolean }).isWellFormed()); + assert.equal(cutBoundedBoundary.includes('\uFFFD'), false); + + const loneInInput = sanitizeAiGeneratedText('abcd\uD800ef\uDC00', 5); + assert.ok((loneInInput as string & { isWellFormed(): boolean }).isWellFormed()); + assert.ok(loneInInput.length <= 5); + + console.log(' ✅ #1853 spend-cap NaN/Infinity, prototype keys, and UTF-16 surrogates verified'); +} + async function run() { console.log('Review AI points job'); testPrompt(); @@ -1103,6 +1384,7 @@ async function run() { testCollect(); await testJob(); await testDosAiClient(); + await testHardening1853(); } run().catch(err => { diff --git a/apps/importer/src/shopee/__tests__/review-values.test.ts b/apps/importer/src/shopee/__tests__/review-values.test.ts index 31258220..9f08c827 100644 --- a/apps/importer/src/shopee/__tests__/review-values.test.ts +++ b/apps/importer/src/shopee/__tests__/review-values.test.ts @@ -258,6 +258,51 @@ async function run() { assert.deepEqual(edge.get(M2)!.criteria_scores, { battery: { score: 4.133, weight_sum: 1.5, writers: 3 }, }); + // Hardening (#1853): NaN / Infinity in criteria_ratings, stars, or weights never reach BigInt, + // and prototype keys (__proto__, constructor, prototype) are ignored. + const protoCriteria = JSON.parse( + '{"__proto__":4,"constructor":4,"prototype":4,"battery":4}' + ) as Record; + const malformedRatings = expectedMasterRatings({ + reviews: [ + review(40, 'l2', { + rating_star: 5, + criteria_ratings: { + ...protoCriteria, + display: Number.NaN, + camera: Number.POSITIVE_INFINITY, + sound: Number.NEGATIVE_INFINITY, + }, + }), + review(41, 'l2', { + rating_star: 4, + criteria_ratings: { ...protoCriteria, display: Number.NaN }, + }), + review(42, 'l2', { + rating_star: Number.NaN, + criteria_ratings: { ...protoCriteria, display: Number.POSITIVE_INFINITY }, + }), + review(43, 'l2', { rating_exact: 'NaN', rating_star: 5 }), + review(44, 'l2', { rating_exact: Number.POSITIVE_INFINITY, rating_star: 5 }), + review(45, 'l2', { helpfulness_weight: Number.POSITIVE_INFINITY, rating_star: 5 }), + review(46, 'l2', { helpfulness_weight: 'NaN', rating_star: 5 }), + review(47, 'l2', { helpfulness_weight: Number.MAX_VALUE, rating_star: 5 }), + ], + listings, + values: new Map(), + }); + const m2Malformed = malformedRatings.get(M2)!; + assert.equal(Object.getPrototypeOf(m2Malformed.criteria_scores), Object.prototype); + assert.equal(Object.hasOwn(m2Malformed.criteria_scores, '__proto__'), false); + assert.equal(Object.hasOwn(m2Malformed.criteria_scores, 'constructor'), false); + assert.equal(Object.hasOwn(m2Malformed.criteria_scores, 'prototype'), false); + assert.deepEqual(Object.keys(m2Malformed.criteria_scores), ['battery']); + assert.equal(m2Malformed.criteria_scores.battery!.writers, 3); + assert.equal( + m2Malformed.rating_weight_sum, + 5, + 'only finite 1..5 star reviews count and huge finite weight clamps cleanly to 3' + ); assert.ok( !masterRatingMatches( { diff --git a/apps/importer/src/shopee/jobs/review-ai-points.ts b/apps/importer/src/shopee/jobs/review-ai-points.ts index 8a013180..f8dc2bc5 100644 --- a/apps/importer/src/shopee/jobs/review-ai-points.ts +++ b/apps/importer/src/shopee/jobs/review-ai-points.ts @@ -301,6 +301,21 @@ export function dosAiReviewAiClient( }; const json = async (path: string, init?: RequestInit, retry?: 'all' | 'throttled') => JSON.parse((await call(path, init, retry)).text) as T; + const normalizePathId = (value: unknown): string => { + if (typeof value !== 'string') throw new Error('Invalid dos.ai resource identifier.'); + const wellFormed = (value as string & { toWellFormed(): string }).toWellFormed().trim(); + if ( + !wellFormed || + wellFormed === '.' || + wellFormed === '..' || + wellFormed.length > 256 || + /[\s/?#\\\p{Cc}]/u.test(wellFormed) + ) { + throw new Error('Invalid dos.ai resource identifier.'); + } + return wellFormed; + }; + const encodePathId = (value: unknown): string => encodeURIComponent(normalizePathId(value)); return { async createBatch(requests) { const form = new FormData(); @@ -313,23 +328,24 @@ export function dosAiReviewAiClient( 'review-ai-points.jsonl' ); const file = await json<{ id: string }>('/files', { method: 'POST', body: form }); + const fileId = normalizePathId(file?.id); const batch = await json<{ id: string }>( '/batches', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - input_file_id: file.id, + input_file_id: fileId, endpoint: '/v1/chat/completions', completion_window: '24h', }), }, 'throttled' ); - return { id: batch.id }; + return { id: normalizePathId(batch?.id) }; }, async batchStatus(id) { - const batch = await json<{ status: string }>(`/batches/${encodeURIComponent(id)}`); + const batch = await json<{ status: string }>(`/batches/${encodePathId(id)}`); if (batch.status === 'cancelling') return 'canceling'; return ['completed', 'failed', 'cancelled', 'expired'].includes(batch.status) ? 'ended' @@ -340,7 +356,7 @@ export function dosAiReviewAiClient( status?: string; output_file_id?: string | null; errmsg?: string | null; - }>(`/batches/${encodeURIComponent(id)}`); + }>(`/batches/${encodePathId(id)}`); if (!batch.output_file_id) { // Failed as a whole, or ended with no line run: nothing to store. throw new BatchGoneError( @@ -350,7 +366,7 @@ export function dosAiReviewAiClient( } let text: string; try { - text = (await call(`/files/${encodeURIComponent(batch.output_file_id)}/content`)).text; + text = (await call(`/files/${encodePathId(batch.output_file_id)}/content`)).text; } catch (err) { if (/ 404 /.test(String(err))) { throw new BatchGoneError(`dos.ai batch ${id}: its output file is gone (404)`, 'expired'); @@ -370,7 +386,7 @@ export function dosAiReviewAiClient( } }, async cancelBatch(id) { - await call(`/batches/${encodeURIComponent(id)}/cancel`, { method: 'POST' }); + await call(`/batches/${encodePathId(id)}/cancel`, { method: 'POST' }); }, }; } @@ -700,7 +716,12 @@ export function collectBatch( const message = result.result.message; for (const key of Object.keys(usage) as (keyof typeof usage)[]) { const n = Number(message.usage?.[key] ?? 0); - usage[key] += Number.isFinite(n) && n > 0 ? n : 0; + if (Number.isFinite(n) && n > 0) { + const next = usage[key] + n; + usage[key] = Number.isFinite(next) + ? Math.min(next, Number.MAX_SAFE_INTEGER) + : Number.MAX_SAFE_INTEGER; + } } if (message.stop_reason === 'end_turn') { const parsed = parseOutput(message) as { reviews?: unknown } | null; @@ -796,8 +817,13 @@ export async function storeAiRows( for (let attempt = 1; ; attempt++) { const { data, error } = await supabase.rpc('store_review_ai_points', { p_rows: part }); if (!error) { - for (const [key, n] of Object.entries((data ?? {}) as Record)) { - if (key in total) total[key as keyof StoreCounts] += Number(n) || 0; + if (data && typeof data === 'object' && !Array.isArray(data)) { + for (const [key, n] of Object.entries(data as Record)) { + if (Object.hasOwn(total, key)) { + const count = Number(n); + total[key as keyof StoreCounts] += Number.isFinite(count) && count > 0 ? count : 0; + } + } } break; } @@ -811,15 +837,54 @@ export async function storeAiRows( return total; } -const monthStart = (now: Date) => - new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)).toISOString(); +const monthStartMs = (now: Date) => Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1); + +function parseBatchUsd(raw: unknown): number | null { + if (raw === null || raw === undefined) return null; + if (typeof raw === 'number') { + return Number.isFinite(raw) ? raw : Number.POSITIVE_INFINITY; + } + if (typeof raw === 'string') { + const trimmed = raw.trim(); + if (!trimmed) return Number.POSITIVE_INFINITY; + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : Number.POSITIVE_INFINITY; + } + return Number.POSITIVE_INFINITY; +} /** This month's spend of a model: reported cost, or the reserved ceiling while open. */ export function monthSpendUsd(batches: readonly AiBatchRow[], model: string, now: Date): number { - const start = monthStart(now); - return batches - .filter(b => b.model === model && b.submitted_at >= start) - .reduce((s, b) => s + Number(b.cost_usd ?? b.reserved_usd ?? 0), 0); + const startMs = monthStartMs(now); + if (!Number.isFinite(startMs) || typeof model !== 'string') return Number.POSITIVE_INFINITY; + const targetModel = model.trim(); + let total = 0; + for (const b of batches) { + if (!b || typeof b !== 'object') return Number.POSITIVE_INFINITY; + const rowModel = typeof b.model === 'string' ? b.model.trim() : ''; + if (rowModel !== targetModel) continue; + const submittedMs = + typeof b.submitted_at === 'string' ? Date.parse(b.submitted_at) : Number.NaN; + if (!Number.isFinite(submittedMs)) return Number.POSITIVE_INFINITY; + if (submittedMs < startMs) continue; + const cost = parseBatchUsd(b.cost_usd); + const reserved = parseBatchUsd(b.reserved_usd); + if (cost === Number.POSITIVE_INFINITY || reserved === Number.POSITIVE_INFINITY) { + return Number.POSITIVE_INFINITY; + } + if (cost === null) { + if (reserved === null || reserved < 0) return Number.POSITIVE_INFINITY; + total += reserved; + continue; + } + if (cost < 0) { + if (reserved === null || reserved < 0) return Number.POSITIVE_INFINITY; + total += reserved; + continue; + } + total += cost; + } + return total; } const BATCH_COLUMNS = @@ -1121,10 +1186,17 @@ export async function runReviewAiPointsJob( ); result.monthSpendUsd = monthSpendUsd(refreshed, model, now); const cap = REVIEW_AI_MODELS[model].monthlyCapUsd; - result.capped = requests.length > 0 && result.monthSpendUsd + result.reservedUsd > cap; + result.capped = + requests.length > 0 && + (!Number.isFinite(result.monthSpendUsd) || + !Number.isFinite(result.reservedUsd) || + result.monthSpendUsd + result.reservedUsd > cap); + const monthText = Number.isFinite(result.monthSpendUsd) + ? `$${result.monthSpendUsd.toFixed(4)}` + : 'non-finite (fail-closed)'; logger.info( 'review_ai.plan', - `${apply ? 'Sending' : 'Would send'} ${chosen.length} of ${candidates.length} due reviews in ${requests.length} requests; ceiling $${result.reservedUsd.toFixed(4)}, month $${result.monthSpendUsd.toFixed(4)} of $${cap}.`, + `${apply ? 'Sending' : 'Would send'} ${chosen.length} of ${candidates.length} due reviews in ${requests.length} requests; ceiling $${result.reservedUsd.toFixed(4)}, month ${monthText} of $${cap}.`, { apply, model, diff --git a/apps/importer/src/shopee/jobs/review-values.ts b/apps/importer/src/shopee/jobs/review-values.ts index c6e5e3e3..18caff94 100644 --- a/apps/importer/src/shopee/jobs/review-values.ts +++ b/apps/importer/src/shopee/jobs/review-values.ts @@ -346,6 +346,7 @@ export interface MasterRatingExpected { } const CRITERION_KEY = /^[a-z_]{2,32}$/; +const DISALLOWED_CRITERION_KEYS = new Set(['__proto__', 'constructor', 'prototype']); /** * P2's rating with P6's weight, per master (cluster_id scope): user reviews @@ -374,14 +375,15 @@ export function expectedMasterRatings(input: { review.reviewer_type !== 'user' || flag === null || flag.include_in_rating === false || - !(weight > 0) + !Number.isFinite(weight) || + weight <= 0 ) { continue; } const exact = review.rating_exact == null ? null : Number(review.rating_exact); const list = byMaster.get(masterId) ?? []; const factor = completenessWeightFactor(input.values.get(review.id)?.weightCompleteness); - const w = toUnits(weight, 100n) * toUnits(factor, V_UNIT); + const w = toUnits(Math.min(weight, 3), 100n) * toUnits(factor, V_UNIT); list.push({ // LEAST(3, helpfulness_weight x factor), as reviewRatingWeight, in millionths. w: w < 3n * W_UNIT ? w : 3n * W_UNIT, @@ -392,14 +394,21 @@ export function expectedMasterRatings(input: { } const out = new Map(); for (const [masterId, eligible] of byMaster) { - const rated = eligible.filter(e => e.star >= 1 && e.star <= 5); + const rated = eligible.filter(e => Number.isFinite(e.star) && e.star >= 1 && e.star <= 5); const sumW = rated.reduce((s, e) => s + e.w, 0n); const sumWS = rated.reduce((s, e) => s + e.w * toUnits(e.star, V_UNIT), 0n); const perKey = new Map(); for (const e of eligible) { if (!e.criteria || typeof e.criteria !== 'object' || Array.isArray(e.criteria)) continue; for (const [key, value] of Object.entries(e.criteria as Record)) { - if (!CRITERION_KEY.test(key) || typeof value !== 'number' || value < 1 || value > 5) { + if ( + !CRITERION_KEY.test(key) || + DISALLOWED_CRITERION_KEYS.has(key) || + typeof value !== 'number' || + !Number.isFinite(value) || + value < 1 || + value > 5 + ) { continue; } const acc = perKey.get(key) ?? { writers: 0, w: 0n, shown: 0n, ws: 0n }; diff --git a/apps/importer/src/shopee/logger.ts b/apps/importer/src/shopee/logger.ts index 538ab5ca..e3524522 100644 --- a/apps/importer/src/shopee/logger.ts +++ b/apps/importer/src/shopee/logger.ts @@ -129,6 +129,22 @@ export function computeRetryDelayMs( return Math.min(safeMaxDelayMs, half + jitter); } +const toWellFormedUtf16 = (value: string): string => + (value as string & { toWellFormed(): string }).toWellFormed(); + +function sliceUtf16WellFormed(value: string, maxUnits: number): string { + if (value.length <= maxUnits) return toWellFormedUtf16(value); + let end = maxUnits; + if (end > 0) { + const prev = value.charCodeAt(end - 1); + const next = value.charCodeAt(end); + if (prev >= 0xd800 && prev <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) { + end -= 1; + } + } + return toWellFormedUtf16(value.slice(0, end)); +} + /** * Sanitizes untrusted AI-generated output text (`ai_summary`, `pros`, `cons`, `reason`) * by stripping HTML tags, control characters, URLs, and phone/Zalo numbers to prevent @@ -140,26 +156,24 @@ export function sanitizeAiGeneratedText(raw: unknown, maxLen = 400): string { typeof maxLen === 'number' && Number.isFinite(maxLen) && maxLen >= 1 ? Math.min(Math.floor(maxLen), MAX_LOG_STRING_LENGTH) : 400; - const boundedInput = raw.slice(0, Math.max(safeMaxLen * 4, 4096)); - return ( - boundedInput - // `[^<>]` rather than `[^>]`: a run of unclosed '<' would otherwise be - // rescanned to the end of the string from each '<' (quadratic). - .replace(/<[^<>]*>/g, ' ') - // Strip explicit http(s):// and www. URLs - .replace(/\b(?:https?:\/\/|www\.)\S+/gi, '[link-removed]') - // Strip bare domain links with common TLDs/paths (e.g. bit.ly/xyz, shopee.vn/abc, t.me/xyz) - .replace( - /\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.(?:vn|com|net|org|io|me|ly|gg|co|shop|store|xyz|info)(?:\/\S*)?\b/gi, - '[link-removed]' - ) - // Strip Vietnamese phone / Zalo numbers (e.g. 0912345678, +84 912 345 678, 098.123.4567) - .replace(/(?:\+?84|0)\s*[1-9](?:[\s.-]*\d){7,9}\b/g, '[phone-removed]') - .replace(/\p{Cc}+/gu, ' ') - .replace(/\s+/g, ' ') - .trim() - .slice(0, safeMaxLen) - ); + const boundedInput = sliceUtf16WellFormed(raw, Math.max(safeMaxLen * 4, 4096)); + const cleaned = boundedInput + // `[^<>]` rather than `[^>]`: a run of unclosed '<' would otherwise be + // rescanned to the end of the string from each '<' (quadratic). + .replace(/<[^<>]*>/g, ' ') + // Strip explicit http(s):// and www. URLs + .replace(/\b(?:https?:\/\/|www\.)\S+/gi, '[link-removed]') + // Strip bare domain links with common TLDs/paths (e.g. bit.ly/xyz, shopee.vn/abc, t.me/xyz) + .replace( + /\b[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.(?:vn|com|net|org|io|me|ly|gg|co|shop|store|xyz|info)(?:\/\S*)?\b/gi, + '[link-removed]' + ) + // Strip Vietnamese phone / Zalo numbers (e.g. 0912345678, +84 912 345 678, 098.123.4567) + .replace(/(?:\+?84|0)\s*[1-9](?:[\s.-]*\d){7,9}\b/g, '[phone-removed]') + .replace(/\p{Cc}+/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); + return sliceUtf16WellFormed(cleaned, safeMaxLen).trimEnd(); } /** diff --git a/apps/importer/src/shopee/product-points-ai-prompt.ts b/apps/importer/src/shopee/product-points-ai-prompt.ts index bd7a8a09..ad67abd1 100644 --- a/apps/importer/src/shopee/product-points-ai-prompt.ts +++ b/apps/importer/src/shopee/product-points-ai-prompt.ts @@ -324,7 +324,9 @@ export function mergeAndSelectAiProductPoints( pros: list.filter(c => c.kind === 'pro').slice(0, max), cons: list.filter(c => c.kind === 'con').slice(0, max), }); - const byCriterion: Record = {}; + const byCriterion: Record = Object.create( + null + ) as Record; for (const key of new Set(rebuilt.map(c => c.criterionKey ?? GENERAL_POINTS))) { byCriterion[key] = pickSides( rebuilt.filter(c => (c.criterionKey ?? GENERAL_POINTS) === key), diff --git a/apps/importer/src/shopee/review-ai-prompt.ts b/apps/importer/src/shopee/review-ai-prompt.ts index 36991a64..84bc0d27 100644 --- a/apps/importer/src/shopee/review-ai-prompt.ts +++ b/apps/importer/src/shopee/review-ai-prompt.ts @@ -366,15 +366,17 @@ export function batchCostUsd( cache_creation_input_tokens?: number | null; } ): number { + if (!isReviewAiModel(model)) return 0; const price = REVIEW_AI_MODELS[model]; - const n = (v: number | null | undefined) => (typeof v === 'number' && v > 0 ? v : 0); + const mTokens = (v: number | null | undefined) => + typeof v === 'number' && Number.isFinite(v) && v > 0 ? v / 1_000_000 : 0; const usd = - (n(usage.input_tokens) * price.input + - n(usage.cache_read_input_tokens) * price.input * CACHE_READ_FACTOR + - n(usage.cache_creation_input_tokens) * price.input * CACHE_WRITE_FACTOR + - n(usage.output_tokens) * price.output) / - 1_000_000; - return usd * BATCH_DISCOUNT; + (mTokens(usage.input_tokens) * price.input + + mTokens(usage.cache_read_input_tokens) * price.input * CACHE_READ_FACTOR + + mTokens(usage.cache_creation_input_tokens) * price.input * CACHE_WRITE_FACTOR + + mTokens(usage.output_tokens) * price.output) * + BATCH_DISCOUNT; + return Number.isFinite(usd) ? usd : Number.POSITIVE_INFINITY; } /**