Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 156 additions & 0 deletions apps/importer/src/shopee/__tests__/backfill-specs-scope.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@ import {
} from '../flagship-seed-data.js';
import {
parseBackfillSpecsArgs,
runBackfillSpecsCli,
runMasterSpecsBackfill,
runMasterSpecsBackfillWith,
runQuickSpecSyncAndAudit,
selectOnlyScope,
selectOnlySeeds,
type BackfillSpecsDeps,
type BackfillSpecsOptions,
} from '../jobs/backfill-specs.js';
import type { ProductSpecifications } from '@repo/shared-data';
import type { RateSupabaseClient } from '../types.js';
Expand Down Expand Up @@ -859,3 +862,156 @@ test('unscoped master scan paginates past 1000 rows via keyset fetchByKey (#1665
'second page queried via keyset gt(id, last)'
);
});

test('--audit-only rejects --apply in either order and never executes apply: true (#1854)', async () => {
assert.deepEqual(parseBackfillSpecsArgs([]), { apply: false, auditOnly: false });
assert.deepEqual(parseBackfillSpecsArgs(['--audit-only']), { apply: false, auditOnly: true });
assert.deepEqual(parseBackfillSpecsArgs(['--apply']), { apply: true, auditOnly: false });
assert.deepEqual(parseBackfillSpecsArgs(['--apply', '--only', APPROVED[0]!, '--expect', '1']), {
apply: true,
auditOnly: false,
only: [APPROVED[0]!],
expect: 1,
});

for (const argv of [
['--audit-only', '--apply'],
['--apply', '--audit-only'],
['--audit-only', '--only', APPROVED[0]!, '--apply', '--expect', '1'],
['--only', APPROVED[0]!, '--apply', '--expect', '1', '--audit-only'],
]) {
assert.throws(
() => parseBackfillSpecsArgs(argv),
/--apply does not combine with --audit-only/,
argv.join(' ')
);
}
assert.throws(
() => parseBackfillSpecsArgs(['--audit-only', '--audit-only']),
/--audit-only is repeated/
);

// CLI dispatcher rejects both flag orders before invoking either runner.
for (const argv of [
['--audit-only', '--apply'],
['--apply', '--audit-only'],
]) {
let called = false;
await assert.rejects(
runBackfillSpecsCli(argv, {
runQuickAudit: async () => {
called = true;
return 0;
},
runBackfill: async () => {
called = true;
return null;
},
}),
/--apply does not combine with --audit-only/,
argv.join(' ')
);
assert.equal(called, false, `${argv.join(' ')} never reaches a runner`);
}

// Defensive guard inside runBackfillSpecsCli even if parseArgs were bypassed.
let quickAuditCalled = false;
await assert.rejects(
runBackfillSpecsCli(['--audit-only'], {
parseArgs: () => ({ apply: true, auditOnly: true }),
runQuickAudit: async () => {
quickAuditCalled = true;
return 0;
},
}),
/--apply does not combine with --audit-only/
);
assert.equal(quickAuditCalled, false);

// Valid --audit-only dispatches with apply: false, auditOnly: true.
const quickCalls: Array<BackfillSpecsOptions & { readonly auditOnly?: boolean }> = [];
const backfillCalls: BackfillSpecsOptions[] = [];
const auditCount = await runBackfillSpecsCli(['--audit-only'], {
runQuickAudit: async opts => {
quickCalls.push(opts);
return 42;
},
runBackfill: async opts => {
backfillCalls.push(opts);
return null;
},
});
assert.equal(auditCount, 42);
assert.deepEqual(quickCalls, [{ apply: false, auditOnly: true }]);
assert.deepEqual(backfillCalls, []);

// Valid scoped --apply dispatches to runBackfill only.
const scopedQuickCalls: Array<BackfillSpecsOptions & { readonly auditOnly?: boolean }> = [];
const scopedBackfillCalls: BackfillSpecsOptions[] = [];
await runBackfillSpecsCli(['--apply', '--only', APPROVED[0]!, '--expect', '1'], {
runQuickAudit: async opts => {
scopedQuickCalls.push(opts);
return 0;
},
runBackfill: async opts => {
scopedBackfillCalls.push(opts);
return null;
},
});
assert.deepEqual(scopedQuickCalls, []);
assert.deepEqual(scopedBackfillCalls, [{ apply: true, only: [APPROVED[0]!], expect: 1 }]);

// Defensive guard at the entry of runQuickSpecSyncAndAudit fails closed before reading env/client.
const envKeys = [
'SUPABASE_URL',
'NEXT_PUBLIC_SUPABASE_URL',
'SUPABASE_SECRET_KEY',
'NEXT_PUBLIC_SUPABASE_ANON_KEY',
'SUPABASE_ANON_KEY',
];
const saved = Object.fromEntries(envKeys.map(k => [k, process.env[k]]));
for (const k of envKeys) delete process.env[k];
try {
for (const badOptions of [
{ auditOnly: true, apply: true },
{ apply: true },
{ apply: true, only: [APPROVED[0]!], expect: 1 },
{ auditOnly: 'true' as unknown as boolean, apply: true },
{ auditOnly: true, apply: 'true' as unknown as boolean },
{ apply: 1 as unknown as boolean },
]) {
await assert.rejects(
runQuickSpecSyncAndAudit(badOptions),
/--apply does not combine with --audit-only/,
JSON.stringify(badOptions)
);
}
for (const badScoped of [
{ auditOnly: true, only: [APPROVED[0]!] },
{ only: [APPROVED[0]!] },
{ auditOnly: true, expect: 1 },
{ expect: 1 },
{ auditOnly: true, expectInserts: 0 },
{ expectInserts: 0 },
]) {
await assert.rejects(
runQuickSpecSyncAndAudit(badScoped),
/--only, --expect and --expect-inserts do not combine with --audit-only/,
JSON.stringify(badScoped)
);
}
for (const badInsertMissing of [
{ auditOnly: true, insertMissing: true },
{ insertMissing: true },
{ insertMissing: 'true' as unknown as boolean },
]) {
await assert.rejects(
runQuickSpecSyncAndAudit(badInsertMissing),
/--insert-missing does not combine with --audit-only/,
JSON.stringify(badInsertMissing)
);
}
} finally {
for (const k of envKeys) if (saved[k] !== undefined) process.env[k] = saved[k];
}
});
127 changes: 75 additions & 52 deletions apps/importer/src/shopee/jobs/backfill-specs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
* between the dry run and the apply cannot pass as an update (DOS/Rate#1739).
* A scoped run only updates existing masters: a named key with no master row stops an
* --apply before any write (a dry run warns). Any unknown argument, `--flag=value` or a
* repeated flag fails. Neither flag combines with --audit-only. The read-only catalog
* repeated flag fails. None of --apply, --only, --expect, --expect-inserts or
* --insert-missing combines with --audit-only. The read-only catalog
* audit still runs at the end, so a scoped --apply can exit 1 after its writes when an
* unrelated master fails the audit: read the log, not only the exit code.
*
Expand Down Expand Up @@ -184,7 +185,7 @@ export function specUpdateChanges(
/**
* The spec columns a seed writes onto its existing master: specs and key specs merged into
* the stored ones, minus the seed's retracted paths, and for a replaced seed its variant
* name and value columns. Shared by the full backfill and the quick sync, so neither keeps a retracted value.
* name and value columns. Ensures no seed update keeps a retracted value.
*/
export function seedMasterSpecUpdate(
existing: { specifications?: unknown; key_specs?: unknown } | null | undefined,
Expand Down Expand Up @@ -578,6 +579,9 @@ export function parseBackfillSpecsArgs(argv: readonly string[]): BackfillSpecsAr
}
expectInserts = Number(expectInsertsRaw.trim());
}
if (auditOnly && apply) {
throw new Error('--apply does not combine with --audit-only');
}
if (auditOnly && (only !== undefined || expect !== undefined || expectInserts !== undefined)) {
throw new Error('--only, --expect and --expect-inserts do not combine with --audit-only');
}
Expand Down Expand Up @@ -1248,11 +1252,28 @@ export async function runMasterSpecsBackfillWith(
};
}

/**
* Read-only quick spec audit (`--audit-only`). Fails closed before loading the Supabase
* client if `apply` or any scoped/write flag is passed, including from delegating callers
* such as `test-ai-enricher.ts --audit-only`.
*/
export async function runQuickSpecSyncAndAudit(
options: BackfillSpecsOptions = {}
options: BackfillSpecsOptions & { readonly auditOnly?: boolean } = {}
): Promise<number> {
if (!options || typeof options !== 'object' || Array.isArray(options)) options = {};
const apply = options.apply === true;
if (options.apply) {
throw new Error('--apply does not combine with --audit-only');
}
if (
options.only !== undefined ||
options.expect !== undefined ||
options.expectInserts !== undefined
) {
throw new Error('--only, --expect and --expect-inserts do not combine with --audit-only');
}
if (options.insertMissing) {
throw new Error('--insert-missing does not combine with --audit-only');
}
const { createClient } = await import('@supabase/supabase-js');
const rawSupabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL || process.env.SUPABASE_URL;
const supabaseKey = (
Expand All @@ -1271,41 +1292,56 @@ export async function runQuickSpecSyncAndAudit(
})
);

if (apply) {
for (const seed of FLAGSHIP_DEVICES_TO_SEED) {
const { data: existingRow, error: lookupErr } = await supabase
.from('master_products')
.select('id, key_specs, specifications')
.eq('model_key', seed.model_key)
.maybeSingle();
if (lookupErr) {
throw new Error(`Failed to look up master product ${seed.model_key}: ${lookupErr.message}`);
}
const { error: upErr } = await supabase
.from('master_products')
.update({
...seedMasterSpecUpdate(
existingRow as { specifications?: unknown; key_specs?: unknown } | null,
seed
),
updated_at: new Date().toISOString(),
})
.eq('model_key', seed.model_key);
if (upErr) {
throw new Error(`Failed to sync specs of ${seed.model_key}: ${upErr.message}`);
}
}
}

const total = await auditMasterSpecs(supabase);
logger.info(
'backfill.audit_passed',
`SPEC CLOUD AUDIT PASSED [mode=${apply ? 'APPLY' : 'AUDIT-ONLY'}]: ${total}/${total} master products on Supabase Cloud verified with 10/10 Field Groups (main camera, and a second camera unless single-camera).`,
{ apply, totalAudited: total }
`SPEC CLOUD AUDIT PASSED [mode=AUDIT-ONLY]: ${total}/${total} master products on Supabase Cloud verified with 10/10 Field Groups (main camera, and a second camera unless single-camera).`,
{ apply: false, totalAudited: total }
);
return total;
}

export interface BackfillSpecsCliRunners {
readonly parseArgs?: (argv: readonly string[]) => BackfillSpecsArgs;
readonly runQuickAudit?: (
options: BackfillSpecsOptions & { readonly auditOnly?: boolean }
) => Promise<unknown>;
readonly runBackfill?: (options: BackfillSpecsOptions) => Promise<unknown>;
}

/**
* Parses CLI arguments and dispatches to the read-only quick audit (`--audit-only`) or the
* master spec backfill. Fails closed if `auditOnly` and `apply` ever appear together.
*/
export async function runBackfillSpecsCli(
argv: readonly string[],
runners: BackfillSpecsCliRunners = {}
): Promise<unknown> {
const parseArgs = runners.parseArgs ?? parseBackfillSpecsArgs;
const args = parseArgs(argv);
if (args.auditOnly) {
if (args.apply) {
throw new Error('--apply does not combine with --audit-only');
}
if (args.only !== undefined || args.expect !== undefined || args.expectInserts !== undefined) {
throw new Error('--only, --expect and --expect-inserts do not combine with --audit-only');
}
if (args.insertMissing) {
throw new Error('--insert-missing does not combine with --audit-only');
}
const runQuickAudit = runners.runQuickAudit ?? runQuickSpecSyncAndAudit;
return runQuickAudit({ apply: false, auditOnly: true });
}
const runBackfill = runners.runBackfill ?? runMasterSpecsBackfill;
return runBackfill({
apply: args.apply,
...(args.only !== undefined ? { only: args.only } : {}),
...(args.expect !== undefined ? { expect: args.expect } : {}),
...(args.expectInserts !== undefined ? { expectInserts: args.expectInserts } : {}),
...(args.insertMissing ? { insertMissing: true } : {}),
});
}

const isDirectExecution =
typeof process !== 'undefined' &&
Array.isArray(process.argv) &&
Expand All @@ -1315,24 +1351,11 @@ const isDirectExecution =
if (isDirectExecution) {
dotenv.config({ quiet: true });
dotenv.config({ path: '../../.env', quiet: true });
Promise.resolve()
.then((): Promise<unknown> => {
const args = parseBackfillSpecsArgs(process.argv.slice(2));
return args.auditOnly
? runQuickSpecSyncAndAudit({ apply: args.apply })
: runMasterSpecsBackfill({
apply: args.apply,
...(args.only !== undefined ? { only: args.only } : {}),
...(args.expect !== undefined ? { expect: args.expect } : {}),
...(args.expectInserts !== undefined ? { expectInserts: args.expectInserts } : {}),
...(args.insertMissing ? { insertMissing: true } : {}),
});
})
.catch((err: unknown) => {
logger.error(
'backfill.fatal',
`Spec backfill/audit execution failed: ${err instanceof Error ? err.message : String(err)}`
);
process.exit(1);
});
runBackfillSpecsCli(process.argv.slice(2)).catch((err: unknown) => {
logger.error(
'backfill.fatal',
`Spec backfill/audit execution failed: ${err instanceof Error ? err.message : String(err)}`
);
process.exit(1);
});
}
Loading