From b5c3257946f38e149f32818dd92c5b44bf420bbd Mon Sep 17 00:00:00 2001 From: Gernot Pansy Date: Thu, 19 Nov 2026 08:38:36 +0100 Subject: [PATCH] Rank tracking: resolve poisoned SERPs and goto links, match real host, log misses - Send resolve_poisoned_results on live and queued checks (extra first page billed) - Send resolve_goto_urls on live checks; live timeout 280s - Match Google /goto results by the real host (url param, breadcrumb) - Second read for missed/absent pairs on the queued path - Log a line for every miss with what came back - Match local grid items by name only when no id is given Co-Authored-By: Claude Sonnet 5.5 --- src/server/db/schema/seo.ts | 2 + src/server/seo/lib/local.ts | 5 +- src/server/seo/lib/rank-tracking.ts | 25 ++++- src/server/seo/lib/seo-lib.test.ts | 37 +++++-- src/server/seo/lib/serp.ts | 30 ++++-- src/server/seo/rank-engine.ts | 161 ++++++++++++++++++++++------ 6 files changed, 205 insertions(+), 55 deletions(-) diff --git a/src/server/db/schema/seo.ts b/src/server/db/schema/seo.ts index e0547b7..2ab3a30 100644 --- a/src/server/db/schema/seo.ts +++ b/src/server/db/schema/seo.ts @@ -215,6 +215,8 @@ export type SeoRankRunStats = { fallbackChecked?: number; /** Live path: keywords already claimed (a crash never re-bills a claimed batch). */ liveCursor?: number; + /** Queued path: 2 once second reads were posted (see phaseReread). */ + reads?: number; }; export const seoRankRuns = pgTable( diff --git a/src/server/seo/lib/local.ts b/src/server/seo/lib/local.ts index 0a3408c..1e849c1 100644 --- a/src/server/seo/lib/local.ts +++ b/src/server/seo/lib/local.ts @@ -314,9 +314,10 @@ export function buildRankGridPoints(center: { latitude: number; longitude: numbe return points; } -/** Match by cid → place_id → title contains name (case-insensitive). */ +/** Match by cid → place_id; title contains name (case-insensitive) only when no id was given. */ export function matchGridItem(items: unknown[], target: { cid?: string; placeId?: string; name?: string }): unknown { - const name = target.name?.toLowerCase(); + // With an id, a title match can be a different business with a similar name. + const name = target.cid || target.placeId ? undefined : target.name?.toLowerCase(); return items.find((item) => { if (target.cid != null && target.cid !== "" && readPath(item, "cid") === target.cid) return true; if (target.placeId != null && target.placeId !== "" && readPath(item, "place_id") === target.placeId) return true; diff --git a/src/server/seo/lib/rank-tracking.ts b/src/server/seo/lib/rank-tracking.ts index 5ef703a..836a8a5 100644 --- a/src/server/seo/lib/rank-tracking.ts +++ b/src/server/seo/lib/rank-tracking.ts @@ -55,6 +55,29 @@ export function costPerSerpAtDepth(depth: number, method: RankCheckMethod): numb : LIVE_BASE_PAGE_COST_USD + (pages - 1) * LIVE_EXTRA_PAGE_COST_USD; } +/** Reads a keyword/device gets in one check: the first, and a second when it got no page back or missed a keyword that ranked. */ +export const MAX_READS_PER_CHECK = 2; + +/** Google serves some scraped searches a fake page that leaves out sites that really rank: a miss on a keyword that ranked is suspect. */ +export function isSuspectMiss(position: number | null | undefined, keywordRanked: boolean): boolean { + return position === null && keywordRanked; +} + +/** `position` undefined = no page came back for the first read. */ +export function needsSecondRead(position: number | null | undefined, keywordRanked: boolean): boolean { + return position === undefined || isSuspectMiss(position, keywordRanked); +} + +/** Queued rank check read: a standard-queue SERP plus the first-page price resolve_poisoned_results adds per task. */ +export function costPerRankCheckRead(depth: number): number { + return costPerSerpAtDepth(depth, "queued") + QUEUED_BASE_PAGE_COST_USD; +} + +/** Live rank check: a live SERP plus the first-page price resolve_poisoned_results adds. */ +export function costPerLiveRankCheck(depth: number): number { + return costPerSerpAtDepth(depth, "live") + LIVE_BASE_PAGE_COST_USD; +} + /** Per metered call rounding (live: 1 check per call; queued: ≤100 checks per task_post). */ export function estimateRankCheckCost(keywordCount: number, devices: RankDevices, depth: number, method: RankCheckMethod) { const totalChecks = keywordCount * devicesCount(devices); @@ -62,7 +85,7 @@ export function estimateRankCheckCost(keywordCount: number, devices: RankDevices let costUsd = 0; for (let offset = 0; offset < totalChecks; offset += perCall) { const checks = Math.min(perCall, totalChecks - offset); - costUsd += roundUsd(checks * costPerSerpAtDepth(depth, method)); + costUsd += roundUsd(checks * (method === "queued" ? costPerRankCheckRead(depth) : costPerLiveRankCheck(depth))); } return { costUsd: roundUsd(costUsd), totalChecks }; } diff --git a/src/server/seo/lib/seo-lib.test.ts b/src/server/seo/lib/seo-lib.test.ts index 3bcacea..27cafd7 100644 --- a/src/server/seo/lib/seo-lib.test.ts +++ b/src/server/seo/lib/seo-lib.test.ts @@ -45,6 +45,8 @@ import { computeNextCheckAt, computeScorecards, estimateRankCheckCost, + isSuspectMiss, + needsSecondRead, estimateScheduledRankCheckCost, normalizeTrackedKeywords, normalizeTrackerDomain, @@ -281,7 +283,7 @@ describe("keyword research payloads + mappers", () => { }); describe("SERP + rank checks", () => { - it("builds analysis + rank check payloads (stop_crawl_on_match, os per device, clamped depth)", () => { + it("builds analysis + rank check payloads (no stop_crawl, os per device, clamped depth)", () => { expect(buildSerpAnalysisTask({ keyword: "k", locationCode: 2840, languageCode: "en", depth: 5 })).toEqual({ keyword: "k", location_code: 2840, @@ -299,8 +301,7 @@ describe("SERP + rank checks", () => { device: "mobile", os: "android", depth: 40, - stop_crawl_on_match: [{ match_value: "example.com", match_type: "with_subdomains" }], - find_targets_in: ["organic"], + resolve_poisoned_results: true, }); const body = buildRankCheckTaskPostBody({ tasks: [ @@ -313,7 +314,7 @@ describe("SERP + rank checks", () => { targetDomain: "example.com", }); expect(body.map((t) => t.tag)).toEqual(["kw-1:desktop", "kw-1:mobile"]); - expect(body[0]).toMatchObject({ location_code: 2840, os: "windows" }); + expect(body[0]).toMatchObject({ location_code: 2840, os: "windows", resolve_poisoned_results: true }); expect(() => buildRankCheckTaskPostBody({ tasks: [], locationCode: 1, languageCode: "en", depth: 10, targetDomain: "x.com" })).toThrow(); }); @@ -344,12 +345,29 @@ describe("SERP + rank checks", () => { ]); }); + it("matches Google /goto results by the real host, not domain google.com", () => { + const input = { keywordId: "k", keyword: "kw", targetDomain: "instahelp.me" }; + const goto = (url: string, breadcrumb?: string) => ({ type: "organic", rank_group: 3, domain: "google.com", url, breadcrumb }); + const viaUrl = goto("https://www.google.com/goto?url=https%3A%2F%2Fwww.instahelp.me%2Fpsychologe%2F"); + expect(buildRankCheckResult(input, [viaUrl]).position).toBe(3); + expect(buildRankCheckResult(input, [goto("https://www.google.com/goto", "https://www.instahelp.me › psychologe")]).position).toBe(3); + expect(buildRankCheckResult(input, [goto("https://www.google.com/goto?url=https%3A%2F%2Fother.com%2F")]).position).toBeNull(); + }); + + it("reads again only when no page came back or a ranked keyword was missed", () => { + expect(needsSecondRead(undefined, false)).toBe(true); + expect(needsSecondRead(null, true)).toBe(true); + expect(needsSecondRead(null, false)).toBe(false); + expect(needsSecondRead(3, true)).toBe(false); + expect(isSuspectMiss(undefined, true)).toBe(false); + }); + it("estimates rank-check costs per metered call", () => { - // live: 1 keyword × 2 devices × (0.002 + 3 × 0.0015) at depth 40 - expect(estimateRankCheckCost(1, "both", 40, "live")).toEqual({ costUsd: 0.013, totalChecks: 2 }); - // queued: 150 checks at depth 10 = 100 + 50 per task_post × 0.0006 - expect(estimateRankCheckCost(150, "desktop", 10, "queued").costUsd).toBeCloseTo(0.09, 5); - expect(estimateScheduledRankCheckCost(50, "mobile", 10, "weekly")).toMatchObject({ checksPerMonth: 4, costUsd: 0.03, monthlyCostUsd: 0.12 }); + // live: 1 keyword × 2 devices × (0.002 + 3 × 0.0015 + 0.002 resolve_poisoned_results) at depth 40 + expect(estimateRankCheckCost(1, "both", 40, "live")).toEqual({ costUsd: 0.017, totalChecks: 2 }); + // queued: 150 checks at depth 10 = 100 + 50 per task_post × 0.0012 (SERP + resolve_poisoned_results) + expect(estimateRankCheckCost(150, "desktop", 10, "queued").costUsd).toBeCloseTo(0.18, 5); + expect(estimateScheduledRankCheckCost(50, "mobile", 10, "weekly")).toMatchObject({ checksPerMonth: 4, costUsd: 0.06, monthlyCostUsd: 0.24 }); }); it("advances schedules on a fixed anchor without drift", () => { @@ -527,6 +545,7 @@ describe("local SEO", () => { const item = matchGridItem([{ title: "Joe's Pizza", cid: "1" }, { title: "Other", cid: "2", place_id: "p2" }], { placeId: "p2" }); expect(item).toMatchObject({ cid: "2" }); expect(matchGridItem([{ title: "Joe's Pizza Berlin" }], { name: "joe's pizza" })).toBeTruthy(); + expect(matchGridItem([{ title: "Joe's Pizza Berlin", cid: "9" }], { cid: "1", name: "joe's pizza" })).toBeUndefined(); expect(summarizeGrid([{ row: 0, col: 0, latitude: 0, longitude: 0, rank: 2 }, { row: 0, col: 1, latitude: 0, longitude: 0, rank: 11 }, { row: 0, col: 2, latitude: 0, longitude: 0, rank: null }])).toEqual({ pointsSearched: 3, pointsFound: 2, diff --git a/src/server/seo/lib/serp.ts b/src/server/seo/lib/serp.ts index 53480bb..2e9e649 100644 --- a/src/server/seo/lib/serp.ts +++ b/src/server/seo/lib/serp.ts @@ -101,14 +101,6 @@ export type RankCheckResult = { type RankLocation = { locationCode: number; languageCode: string; locationName?: string | null }; -/** Stop crawling once the target is found (only pages crawled are billed). */ -function stopCrawlOnTarget(domain: string) { - return { - stop_crawl_on_match: [{ match_value: domain, match_type: "with_subdomains" }], - find_targets_in: ["organic"], - }; -} - function locationParams(loc: RankLocation) { return loc.locationName ? { location_name: loc.locationName } : { location_code: loc.locationCode }; } @@ -123,7 +115,9 @@ export function buildRankCheckTask( device: input.device, os: input.device === "desktop" ? "windows" : "android", depth: clampSerpDepth(input.depth), - ...stopCrawlOnTarget(input.targetDomain), + // DataForSEO swaps in Google's plain web results when Google served a poisoned page (goto links, wrong sites). + // Costs one more first-page SERP per check (see costPerRankCheck* in rank-tracking). + resolve_poisoned_results: true, }; } @@ -144,6 +138,19 @@ export function buildRankCheckTaskPostBody( })); } +/** Google's /goto redirect results report domain "google.com": the real host is in the `url` param, else the breadcrumb. */ +export function organicHost(i: SerpLiveItem): string | undefined { + if (i.domain && !/^(www\.)?google\./i.test(i.domain)) return i.domain; + try { + const u = new URL(i.url ?? ""); + const real = u.searchParams.get("url") ?? u.searchParams.get("q"); + if (/^(www\.)?google\./i.test(u.hostname) && real) return new URL(real).hostname; + } catch { + // not a parsable URL: fall through to the breadcrumb + } + return i.breadcrumb?.match(/^(?:https?:\/\/)?([^\s/›]+)/)?.[1] ?? i.domain ?? undefined; +} + /** * First organic item whose domain equals the target or is a subdomain of it. * position = rank_group (organic-only rank) ?? rank_absolute. serpFeatures = unique item types. @@ -155,8 +162,9 @@ export function buildRankCheckResult( const target = input.targetDomain.toLowerCase(); const list = items.filter((i): i is SerpLiveItem => i != null); const match = list.find((i) => { - if (i.type !== "organic" || i.domain == null) return false; - const d = i.domain.toLowerCase(); + if (i.type !== "organic") return false; + const d = organicHost(i)?.toLowerCase(); + if (!d) return false; return d === target || d.endsWith(`.${target}`); }); return { diff --git a/src/server/seo/rank-engine.ts b/src/server/seo/rank-engine.ts index 61e20c2..3be7df3 100644 --- a/src/server/seo/rank-engine.ts +++ b/src/server/seo/rank-engine.ts @@ -1,5 +1,5 @@ import "server-only"; -import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; +import { and, asc, eq, inArray, isNull, or, sql } from "drizzle-orm"; import { db } from "@/server/db/client"; import { projects, seoRankConfigs, seoRankKeywords, seoRankRuns, seoRankSnapshots, seoRankTasks } from "@/server/db/schema"; import { enqueueJob } from "@/server/jobs/queue"; @@ -18,13 +18,16 @@ import { } from "./rank-tracking"; import { computeNextCheckAt, - costPerSerpAtDepth, + costPerLiveRankCheck, devicesCount, devicesList, estimateRankCheckCost, isScheduledInterval, + isSuspectMiss, KEYWORDS_PER_BATCH, + MAX_READS_PER_CHECK, MAX_TASKS_PER_POST, + needsSecondRead, QUEUED_POLL_INTERVALS_MIN, SCHEDULED_TASK_UNIT_BUDGET, TASK_GET_CONCURRENCY, @@ -33,6 +36,7 @@ import { } from "./lib/rank-tracking"; import { buildRankCheckResult, + organicHost, buildRankCheckTask, buildRankCheckTaskPostBody, isNoResultsMessage, @@ -100,7 +104,12 @@ async function insertSnapshots(runId: string, results: (RankCheckResult & { devi serpFeatures: r.serpFeatures, })), ) - .onConflictDoNothing(); + // A second read only replaces a miss, or a worse position, with a better one. + .onConflictDoUpdate({ + target: [seoRankSnapshots.runId, seoRankSnapshots.trackingKeywordId, seoRankSnapshots.device], + set: { position: sql`excluded.position`, url: sql`excluded.url`, serpFeatures: sql`excluded.serp_features` }, + setWhere: sql`excluded.position IS NOT NULL AND (${seoRankSnapshots.position} IS NULL OR excluded.position < ${seoRankSnapshots.position})`, + }); } async function refreshProgress(runId: string) { @@ -117,6 +126,14 @@ async function enqueueContinuation(run: RankRun, type: string, delayMs = 0) { /* ───────────────────────────── Live checks ───────────────────────────── */ +/** A miss leaves no trace otherwise: log what came back so "not found" can be told from "empty / wrong SERP". */ +function logMiss(runId: string, r: RankCheckResult, device: Device, domain: string, items: SerpLiveItem[]) { + if (r.position !== null) return; + const organic = items.filter((i) => i?.type === "organic"); + const top = organic.slice(0, 5).map((i) => `${organicHost(i) ?? "?"} <${i.domain}> url=${i.url?.slice(0, 120)} crumb=${i.breadcrumb?.slice(0, 60)}`).join(" | "); + console.log(`[rank-check] ${runId} miss keyword="${r.keyword}" device=${device} target=${domain} items=${items.length} organic=${organic.length} top=[${top}]`); +} + async function checkPairsLive(ctx: SeoContext, run: RankRun, config: RankConfig, pairs: Pair[]): Promise { if (await rankChecksUseAi(ctx.workspaceId)) return checkPairsViaAi(ctx, run, config, pairs); const settled = await Promise.allSettled( @@ -124,20 +141,27 @@ async function checkPairsLive(ctx: SeoContext, run: RankRun, config: RankConfig, const task = await dfsLive<{ items?: SerpLiveItem[] | null }>( ctx, SERP_LIVE_PATH, - buildRankCheckTask({ - keyword: p.keyword, - locationCode: config.locationCode, - languageCode: config.languageCode, - locationName: config.locationName, - device: p.device, - targetDomain: config.domain, - depth: config.serpDepth, - }), + { + ...buildRankCheckTask({ + keyword: p.keyword, + locationCode: config.locationCode, + languageCode: config.languageCode, + locationName: config.locationName, + device: p.device, + targetDomain: config.domain, + depth: config.serpDepth, + }), + // Poisoned SERPs link results through google.com/goto: resolve them so the stored URL is the real page. + resolve_goto_urls: true, + }, // Billed → no 5xx retries; "No Search Results" (40501) is a valid empty SERP. - { feature: "rank_tracking", treatNoResultsAsEmpty: true, retry5xx: false, estimatedCostUsd: costPerSerpAtDepth(config.serpDepth, "live") }, + { feature: "rank_tracking", treatNoResultsAsEmpty: true, retry5xx: false, estimatedCostUsd: costPerLiveRankCheck(config.serpDepth), timeoutMs: 280_000 }, ); await addRunCost(run.id, task.cost); - return { ...buildRankCheckResult({ keywordId: p.keywordId, keyword: p.keyword, targetDomain: config.domain }, task.result[0]?.items ?? []), device: p.device }; + const items = task.result[0]?.items ?? []; + const r = buildRankCheckResult({ keywordId: p.keywordId, keyword: p.keyword, targetDomain: config.domain }, items); + logMiss(run.id, r, p.device, config.domain, items); + return { ...r, device: p.device }; }), ); const results: (RankCheckResult & { device: Device })[] = []; @@ -218,16 +242,12 @@ async function phaseLive(ctx: SeoContext, run: RankRun, config: RankConfig, dead return "done"; } -/** task_post in chunks of ≤100 pairs; rows are claimed before posting; rejected / failed chunks → live fallback. */ -async function phasePost(ctx: SeoContext, run: RankRun, config: RankConfig) { - const keywords = await runKeywords(run); - const pairs = expand(keywords, config.devices); - const existing = await db - .select({ trackingKeywordId: seoRankTasks.trackingKeywordId, device: seoRankTasks.device }) - .from(seoRankTasks) - .where(eq(seoRankTasks.runId, run.id)); - const done = new Set(existing.map((e) => rankTaskTag(e.trackingKeywordId, e.device))); - const todo = pairs.filter((p) => !done.has(rankTaskTag(p.keywordId, p.device))); +/** + * task_post in chunks of ≤100 pairs; rows are claimed before posting. Rejected / failed chunks go to the live + * fallback, or for second reads (`onFail: "failed"`) are dropped: the first read's result stands. + */ +async function postPairs(ctx: SeoContext, run: RankRun, config: RankConfig, todo: Pair[], onFail: "fallback" | "failed"): Promise { + const failed = { status: onFail } as const; let queued = 0; for (let i = 0; i < todo.length; i += MAX_TASKS_PER_POST) { const chunk = todo.slice(i, i + MAX_TASKS_PER_POST); @@ -263,7 +283,7 @@ async function phasePost(ctx: SeoContext, run: RankRun, config: RankConfig) { const id = accepted.get(rankTaskTag(p.keywordId, p.device)); await db .update(seoRankTasks) - .set(id ? { taskId: id } : { status: "fallback", message: "Rejected by task_post" }) + .set(id ? { taskId: id } : { ...failed, message: "Rejected by task_post" }) .where(and(eq(seoRankTasks.runId, run.id), eq(seoRankTasks.trackingKeywordId, p.keywordId), eq(seoRankTasks.device, p.device))); if (id) queued++; } @@ -274,7 +294,7 @@ async function phasePost(ctx: SeoContext, run: RankRun, config: RankConfig) { await setRunErrorIfEmpty(run.id, err instanceof Error ? err.message : String(err)); await db .update(seoRankTasks) - .set({ status: "fallback", message: "task_post failed" }) + .set({ ...failed, message: "task_post failed" }) .where( and( eq(seoRankTasks.runId, run.id), @@ -288,11 +308,87 @@ async function phasePost(ctx: SeoContext, run: RankRun, config: RankConfig) { ); } } + return queued; +} + +async function phasePost(ctx: SeoContext, run: RankRun, config: RankConfig) { + const keywords = await runKeywords(run); + const pairs = expand(keywords, config.devices); + const existing = await db + .select({ trackingKeywordId: seoRankTasks.trackingKeywordId, device: seoRankTasks.device }) + .from(seoRankTasks) + .where(eq(seoRankTasks.runId, run.id)); + const done = new Set(existing.map((e) => rankTaskTag(e.trackingKeywordId, e.device))); + const queued = await postPairs(ctx, run, config, pairs.filter((p) => !done.has(rankTaskTag(p.keywordId, p.device))), "fallback"); const stats = { ...(run.stats ?? {}), queueTasks: (run.stats?.queueTasks ?? 0) + queued }; await updateRun(run.id, { phase: "collect", collectRound: 0, stats }); await enqueueContinuation({ ...run, stats }, RANK_COLLECT_JOB, QUEUED_POLL_INTERVALS_MIN[0] * 60_000); } +/** + * This run's positions per "keywordId:device" (null = missed, absent = no page came back) and the keywords that + * ranked in an earlier completed check or on either device in this run: a miss on one of those is suspect. + */ +async function loadReadState(run: RankRun) { + const snaps = await db + .select({ keywordId: seoRankSnapshots.trackingKeywordId, device: seoRankSnapshots.device, position: seoRankSnapshots.position }) + .from(seoRankSnapshots) + .where(eq(seoRankSnapshots.runId, run.id)); + const earlier = await db + .selectDistinct({ keywordId: seoRankSnapshots.trackingKeywordId }) + .from(seoRankSnapshots) + .innerJoin(seoRankRuns, eq(seoRankRuns.id, seoRankSnapshots.runId)) + .where(and(eq(seoRankRuns.configId, run.configId), eq(seoRankRuns.status, "completed"), sql`${seoRankSnapshots.position} IS NOT NULL`)); + const positions = new Map(snaps.map((s) => [rankTaskTag(s.keywordId, s.device), s.position])); + const ranked = new Set([...earlier.map((e) => e.keywordId), ...snaps.filter((s) => s.position !== null).map((s) => s.keywordId)]); + return { positions, ranked }; +} + +/** + * Google serves some scraped searches a fake results page that leaves out sites that really rank. DataForSEO + * replaces most (resolve_poisoned_results); a pair that got no page back, or missed a keyword that ranked before, + * is read once more through the queue. The best position wins. Returns false when no pair needs it. + */ +async function phaseReread(ctx: SeoContext, run: RankRun, config: RankConfig): Promise { + if ((run.stats?.reads ?? 1) >= MAX_READS_PER_CHECK) return false; + const { positions, ranked } = await loadReadState(run); + const todo = expand(await runKeywords(run), config.devices).filter((p) => + needsSecondRead(positions.get(rankTaskTag(p.keywordId, p.device)), ranked.has(p.keywordId)), + ); + if (!todo.length) return false; + console.log(`[rank-check] ${run.id} read 2: ${todo.length} check(s)`); + // Claim first: reads is set before posting so a crash never bills a second read twice. + const stats = { ...(run.stats ?? {}), reads: 2 }; + await updateRun(run.id, { stats }); + for (let i = 0; i < todo.length; i += MAX_TASKS_PER_POST) { + await db + .update(seoRankTasks) + .set({ taskId: null, status: "pending", message: null }) + .where( + and( + eq(seoRankTasks.runId, run.id), + or(...todo.slice(i, i + MAX_TASKS_PER_POST).map((p) => and(eq(seoRankTasks.trackingKeywordId, p.keywordId), eq(seoRankTasks.device, p.device)))), + ), + ); + } + const queued = await postPairs(ctx, run, config, todo, "failed"); + await updateRun(run.id, { phase: "collect", collectRound: 0, stats: { ...stats, queueTasks: (stats.queueTasks ?? 0) + queued } }); + await enqueueContinuation(run, RANK_COLLECT_JOB, QUEUED_POLL_INTERVALS_MIN[0] * 60_000); + return true; +} + +/** After second reads: a miss on a keyword that ranked is dropped, so the pair reads as not checked and keeps its previous position. */ +async function dropSuspectMisses(run: RankRun) { + const { positions, ranked } = await loadReadState(run); + for (const [tag, position] of positions) { + const [keywordId, device] = tag.split(":") as [string, Device]; + if (!isSuspectMiss(position, ranked.has(keywordId))) continue; + await db + .delete(seoRankSnapshots) + .where(and(eq(seoRankSnapshots.runId, run.id), eq(seoRankSnapshots.trackingKeywordId, keywordId), eq(seoRankSnapshots.device, device), isNull(seoRankSnapshots.position))); + } +} + type TaskOutcome = { status: "pending" } | { status: "failed"; message: string } | { status: "completed"; items: SerpLiveItem[] }; async function fetchTaskOutcome(ctx: SeoContext, taskId: string): Promise { @@ -346,13 +442,12 @@ export async function collectQueuedRound(runId: string) { if (o.status === "rejected" || o.value.status === "pending") continue; if (o.value.status === "failed") { await setRunErrorIfEmpty(run.id, o.value.message); - await db.update(seoRankTasks).set({ status: "fallback", message: o.value.message }).where(eq(seoRankTasks.id, t.id)); + await db.update(seoRankTasks).set({ status: run.stats?.reads === 2 ? "failed" : "fallback", message: o.value.message }).where(eq(seoRankTasks.id, t.id)); continue; } - results.push({ - ...buildRankCheckResult({ keywordId: t.trackingKeywordId, keyword: t.keyword, targetDomain: config.domain }, o.value.items), - device: t.device, - }); + const r = buildRankCheckResult({ keywordId: t.trackingKeywordId, keyword: t.keyword, targetDomain: config.domain }, o.value.items); + logMiss(run.id, r, t.device, config.domain, o.value.items); + results.push({ ...r, device: t.device }); await db.update(seoRankTasks).set({ status: "completed" }).where(eq(seoRankTasks.id, t.id)); } await insertSnapshots(run.id, results); @@ -372,7 +467,7 @@ export async function collectQueuedRound(runId: string) { // Window closed (≈15 min): everything unfinished — plus claimed-but-never-posted rows — goes to the live fallback. await db .update(seoRankTasks) - .set({ status: "fallback", message: "Not collected within the polling window" }) + .set({ status: run.stats?.reads === 2 ? "failed" : "fallback", message: "Not collected within the polling window" }) .where(and(eq(seoRankTasks.runId, run.id), eq(seoRankTasks.status, "pending"))); await updateRun(run.id, { phase: "fallback", stats }); await enqueueContinuation(run, RANK_CHECK_JOB); @@ -471,6 +566,8 @@ export async function executeRankCheck(runId: string) { await enqueueContinuation(fresh, RANK_CHECK_JOB); return { continued: true }; } + if (run.method === "queued" && !(await rankChecksUseAi(ctx.workspaceId)) && (await phaseReread(ctx, fresh, config))) return { reread: true }; + if (fresh.stats?.reads === 2) await dropSuspectMisses(fresh); await finalize(fresh); return { done: true }; }