Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/server/db/schema/seo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,8 @@ export type SeoRankRunStats = {
fallbackChecked?: number;
/** Live path: keywords already claimed (a crash never re-bills a claimed batch). */
liveCursor?: number;
/** Queued path: 2 once second reads were posted (see phaseReread). */
reads?: number;
};

export const seoRankRuns = pgTable(
Expand Down
5 changes: 3 additions & 2 deletions src/server/seo/lib/local.ts
Original file line number Diff line number Diff line change
Expand Up @@ -314,9 +314,10 @@ export function buildRankGridPoints(center: { latitude: number; longitude: numbe
return points;
}

/** Match by cid → place_id → title contains name (case-insensitive). */
/** Match by cid → place_id; title contains name (case-insensitive) only when no id was given. */
export function matchGridItem(items: unknown[], target: { cid?: string; placeId?: string; name?: string }): unknown {
const name = target.name?.toLowerCase();
// With an id, a title match can be a different business with a similar name.
const name = target.cid || target.placeId ? undefined : target.name?.toLowerCase();
return items.find((item) => {
if (target.cid != null && target.cid !== "" && readPath(item, "cid") === target.cid) return true;
if (target.placeId != null && target.placeId !== "" && readPath(item, "place_id") === target.placeId) return true;
Expand Down
25 changes: 24 additions & 1 deletion src/server/seo/lib/rank-tracking.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,37 @@ export function costPerSerpAtDepth(depth: number, method: RankCheckMethod): numb
: LIVE_BASE_PAGE_COST_USD + (pages - 1) * LIVE_EXTRA_PAGE_COST_USD;
}

/** Reads a keyword/device gets in one check: the first, and a second when it got no page back or missed a keyword that ranked. */
export const MAX_READS_PER_CHECK = 2;

/** Google serves some scraped searches a fake page that leaves out sites that really rank: a miss on a keyword that ranked is suspect. */
export function isSuspectMiss(position: number | null | undefined, keywordRanked: boolean): boolean {
return position === null && keywordRanked;
}

/** `position` undefined = no page came back for the first read. */
export function needsSecondRead(position: number | null | undefined, keywordRanked: boolean): boolean {
return position === undefined || isSuspectMiss(position, keywordRanked);
}

/** Queued rank check read: a standard-queue SERP plus the first-page price resolve_poisoned_results adds per task. */
export function costPerRankCheckRead(depth: number): number {
return costPerSerpAtDepth(depth, "queued") + QUEUED_BASE_PAGE_COST_USD;
}

/** Live rank check: a live SERP plus the first-page price resolve_poisoned_results adds. */
export function costPerLiveRankCheck(depth: number): number {
return costPerSerpAtDepth(depth, "live") + LIVE_BASE_PAGE_COST_USD;
}

/** Per metered call rounding (live: 1 check per call; queued: ≤100 checks per task_post). */
export function estimateRankCheckCost(keywordCount: number, devices: RankDevices, depth: number, method: RankCheckMethod) {
const totalChecks = keywordCount * devicesCount(devices);
const perCall = method === "queued" ? MAX_TASKS_PER_POST : 1;
let costUsd = 0;
for (let offset = 0; offset < totalChecks; offset += perCall) {
const checks = Math.min(perCall, totalChecks - offset);
costUsd += roundUsd(checks * costPerSerpAtDepth(depth, method));
costUsd += roundUsd(checks * (method === "queued" ? costPerRankCheckRead(depth) : costPerLiveRankCheck(depth)));
}
return { costUsd: roundUsd(costUsd), totalChecks };
}
Expand Down
37 changes: 28 additions & 9 deletions src/server/seo/lib/seo-lib.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ import {
computeNextCheckAt,
computeScorecards,
estimateRankCheckCost,
isSuspectMiss,
needsSecondRead,
estimateScheduledRankCheckCost,
normalizeTrackedKeywords,
normalizeTrackerDomain,
Expand Down Expand Up @@ -281,7 +283,7 @@ describe("keyword research payloads + mappers", () => {
});

describe("SERP + rank checks", () => {
it("builds analysis + rank check payloads (stop_crawl_on_match, os per device, clamped depth)", () => {
it("builds analysis + rank check payloads (no stop_crawl, os per device, clamped depth)", () => {
expect(buildSerpAnalysisTask({ keyword: "k", locationCode: 2840, languageCode: "en", depth: 5 })).toEqual({
keyword: "k",
location_code: 2840,
Expand All @@ -299,8 +301,7 @@ describe("SERP + rank checks", () => {
device: "mobile",
os: "android",
depth: 40,
stop_crawl_on_match: [{ match_value: "example.com", match_type: "with_subdomains" }],
find_targets_in: ["organic"],
resolve_poisoned_results: true,
});
const body = buildRankCheckTaskPostBody({
tasks: [
Expand All @@ -313,7 +314,7 @@ describe("SERP + rank checks", () => {
targetDomain: "example.com",
});
expect(body.map((t) => t.tag)).toEqual(["kw-1:desktop", "kw-1:mobile"]);
expect(body[0]).toMatchObject({ location_code: 2840, os: "windows" });
expect(body[0]).toMatchObject({ location_code: 2840, os: "windows", resolve_poisoned_results: true });
expect(() => buildRankCheckTaskPostBody({ tasks: [], locationCode: 1, languageCode: "en", depth: 10, targetDomain: "x.com" })).toThrow();
});

Expand Down Expand Up @@ -344,12 +345,29 @@ describe("SERP + rank checks", () => {
]);
});

it("matches Google /goto results by the real host, not domain google.com", () => {
const input = { keywordId: "k", keyword: "kw", targetDomain: "instahelp.me" };
const goto = (url: string, breadcrumb?: string) => ({ type: "organic", rank_group: 3, domain: "google.com", url, breadcrumb });
const viaUrl = goto("https://www.google.com/goto?url=https%3A%2F%2Fwww.instahelp.me%2Fpsychologe%2F");
expect(buildRankCheckResult(input, [viaUrl]).position).toBe(3);
expect(buildRankCheckResult(input, [goto("https://www.google.com/goto", "https://www.instahelp.me › psychologe")]).position).toBe(3);
expect(buildRankCheckResult(input, [goto("https://www.google.com/goto?url=https%3A%2F%2Fother.com%2F")]).position).toBeNull();
});

it("reads again only when no page came back or a ranked keyword was missed", () => {
expect(needsSecondRead(undefined, false)).toBe(true);
expect(needsSecondRead(null, true)).toBe(true);
expect(needsSecondRead(null, false)).toBe(false);
expect(needsSecondRead(3, true)).toBe(false);
expect(isSuspectMiss(undefined, true)).toBe(false);
});

it("estimates rank-check costs per metered call", () => {
// live: 1 keyword × 2 devices × (0.002 + 3 × 0.0015) at depth 40
expect(estimateRankCheckCost(1, "both", 40, "live")).toEqual({ costUsd: 0.013, totalChecks: 2 });
// queued: 150 checks at depth 10 = 100 + 50 per task_post × 0.0006
expect(estimateRankCheckCost(150, "desktop", 10, "queued").costUsd).toBeCloseTo(0.09, 5);
expect(estimateScheduledRankCheckCost(50, "mobile", 10, "weekly")).toMatchObject({ checksPerMonth: 4, costUsd: 0.03, monthlyCostUsd: 0.12 });
// live: 1 keyword × 2 devices × (0.002 + 3 × 0.0015 + 0.002 resolve_poisoned_results) at depth 40
expect(estimateRankCheckCost(1, "both", 40, "live")).toEqual({ costUsd: 0.017, totalChecks: 2 });
// queued: 150 checks at depth 10 = 100 + 50 per task_post × 0.0012 (SERP + resolve_poisoned_results)
expect(estimateRankCheckCost(150, "desktop", 10, "queued").costUsd).toBeCloseTo(0.18, 5);
expect(estimateScheduledRankCheckCost(50, "mobile", 10, "weekly")).toMatchObject({ checksPerMonth: 4, costUsd: 0.06, monthlyCostUsd: 0.24 });
});

it("advances schedules on a fixed anchor without drift", () => {
Expand Down Expand Up @@ -527,6 +545,7 @@ describe("local SEO", () => {
const item = matchGridItem([{ title: "Joe's Pizza", cid: "1" }, { title: "Other", cid: "2", place_id: "p2" }], { placeId: "p2" });
expect(item).toMatchObject({ cid: "2" });
expect(matchGridItem([{ title: "Joe's Pizza Berlin" }], { name: "joe's pizza" })).toBeTruthy();
expect(matchGridItem([{ title: "Joe's Pizza Berlin", cid: "9" }], { cid: "1", name: "joe's pizza" })).toBeUndefined();
expect(summarizeGrid([{ row: 0, col: 0, latitude: 0, longitude: 0, rank: 2 }, { row: 0, col: 1, latitude: 0, longitude: 0, rank: 11 }, { row: 0, col: 2, latitude: 0, longitude: 0, rank: null }])).toEqual({
pointsSearched: 3,
pointsFound: 2,
Expand Down
30 changes: 19 additions & 11 deletions src/server/seo/lib/serp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,6 @@ export type RankCheckResult = {

type RankLocation = { locationCode: number; languageCode: string; locationName?: string | null };

/** Stop crawling once the target is found (only pages crawled are billed). */
function stopCrawlOnTarget(domain: string) {
return {
stop_crawl_on_match: [{ match_value: domain, match_type: "with_subdomains" }],
find_targets_in: ["organic"],
};
}

function locationParams(loc: RankLocation) {
return loc.locationName ? { location_name: loc.locationName } : { location_code: loc.locationCode };
}
Expand All @@ -123,7 +115,9 @@ export function buildRankCheckTask(
device: input.device,
os: input.device === "desktop" ? "windows" : "android",
depth: clampSerpDepth(input.depth),
...stopCrawlOnTarget(input.targetDomain),
// DataForSEO swaps in Google's plain web results when Google served a poisoned page (goto links, wrong sites).
// Costs one more first-page SERP per check (see costPerRankCheck* in rank-tracking).
resolve_poisoned_results: true,
};
}

Expand All @@ -144,6 +138,19 @@ export function buildRankCheckTaskPostBody(
}));
}

/** Google's /goto redirect results report domain "google.com": the real host is in the `url` param, else the breadcrumb. */
export function organicHost(i: SerpLiveItem): string | undefined {
if (i.domain && !/^(www\.)?google\./i.test(i.domain)) return i.domain;
try {
const u = new URL(i.url ?? "");
const real = u.searchParams.get("url") ?? u.searchParams.get("q");
if (/^(www\.)?google\./i.test(u.hostname) && real) return new URL(real).hostname;
} catch {
// not a parsable URL: fall through to the breadcrumb
}
return i.breadcrumb?.match(/^(?:https?:\/\/)?([^\s/›]+)/)?.[1] ?? i.domain ?? undefined;
}

/**
* First organic item whose domain equals the target or is a subdomain of it.
* position = rank_group (organic-only rank) ?? rank_absolute. serpFeatures = unique item types.
Expand All @@ -155,8 +162,9 @@ export function buildRankCheckResult(
const target = input.targetDomain.toLowerCase();
const list = items.filter((i): i is SerpLiveItem => i != null);
const match = list.find((i) => {
if (i.type !== "organic" || i.domain == null) return false;
const d = i.domain.toLowerCase();
if (i.type !== "organic") return false;
const d = organicHost(i)?.toLowerCase();
if (!d) return false;
return d === target || d.endsWith(`.${target}`);
});
return {
Expand Down
Loading