Skip to content

fix(browser): focus effort slider with verified pointer fallback #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
AKzar1el wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix(browser): focus effort slider with verified pointer fallback #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 46 additions & 11 deletions browser-companion/src/session.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,22 +332,57 @@ export async function moveChatGptEffortSlider(
let state = await readChatGptEffortSnapshot(menu.sliderContainer);
const { min, max } = state;
if (target < min || target > max || !state.available[target - min]) throw new Error("ChatGPT locks the requested effort");
const control = menu.slider.locator("xpath=ancestor::*[@role='menuitem'][1]");
const menuItem = menu.slider.locator("xpath=ancestor::*[@role='menuitem'][1]");
const thumbFocusable = await menu.slider.evaluate(element => {
if (element.getAttribute("aria-hidden") === "true") return false;
const index = element.getAttribute("tabindex");
return index !== null ? Number(index) >= 0 : element.matches("input, button, select, textarea, a[href]");
}).catch(() => false);
while (state.value !== target) {
if (signal?.aborted) throw new DOMException("Browser action aborted", "AbortError");
const direction = Math.sign(target - state.value);
const before = state.value;
await control.press(direction > 0 ? "ArrowRight" : "ArrowLeft", { timeout:5_000, signal });
const deadline = Date.now() + 5_000;
do {
state = await readChatGptEffortSnapshot(menu.sliderContainer);
if (state.min !== min || state.max !== max || !state.available[target - min]) throw new Error("ChatGPT changed its effort range or availability");
if (state.value !== before) break;
await new Promise(resolve => setTimeout(resolve, 50));
} while (Date.now() < deadline);
if ((state.value - before) * direction <= 0 || (state.value - target) * direction > 0) {
throw new Error("ChatGPT did not move its effort slider toward the requested position");
const moved = async (): Promise<boolean> => {
const deadline = Date.now() + 300;
do {
state = await readChatGptEffortSnapshot(menu.sliderContainer);
if (state.min !== min || state.max !== max || !state.available[target - min]) {
throw new Error("ChatGPT changed its effort range or availability");
}
if (state.value !== before) {
if ((state.value - before) * direction <= 0 || (state.value - target) * direction > 0) {
throw new Error("ChatGPT moved its effort slider outside the requested range");
}
return true;
}
if (signal?.aborted) throw new DOMException("Browser action aborted", "AbortError");
await new Promise(resolve => setTimeout(resolve, 50));
} while (Date.now() < deadline);
return false;
};
const key = direction > 0 ? "ArrowRight" : "ArrowLeft";
let advanced = false;
// The current picker has a real slider thumb. The enclosing menu item is
// only a compatibility fallback for older layouts that own the key handler.
for (const control of thumbFocusable ? [menu.slider, menuItem] : [menuItem]) {
if (signal?.aborted) throw new DOMException("Browser action aborted", "AbortError");
await control.press(key, { timeout:1_500, signal }).catch(() => {});
if (await moved()) { advanced = true; break; }
}
if (!advanced) {
// Some picker layouts ignore both keyboard targets but accept a trusted
// pointer event on the visible track. Do not synthesize ARIA value changes.
const track = menu.sliderContainer.locator('[data-orientation="horizontal"]').filter({visible:true}).first();
const box = await track.boundingBox().catch(() => null);
if (box && box.width >= 28 && max > min) {
const margin = Math.min(14, box.width / 4);
const ratio = (target - min) / (max - min);
const position = {x:margin + (box.width - 2 * margin) * ratio, y:box.height / 2};
await track.click({position,timeout:1_500,signal}).catch(() => {});
advanced = await moved();
}
}
if (!advanced) throw new Error("ChatGPT did not move its effort slider toward the requested position");
}
}

Expand Down
58 changes: 51 additions & 7 deletions browser-companion/tests/capability-discovery.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,35 @@ const fixture = `<!doctype html><style>#prompt-textarea{min-height:24px}[data-mo
const control=document.getElementById('effort'), picker=document.getElementById('picker'), range=document.getElementById('range');
function render() {
const model=definitions[family], index=model.value-model.min;
range.innerHTML='<div data-model-reasoning-effort-slider><span role="slider" aria-valuemin="'+model.min+'" aria-valuemax="'+(model.min+model.labels.length-1)+'" aria-valuenow="'+model.value+'"></span>'+model.labels.map((label,i)=>'<span data-selected="'+(i===index)+'" data-locked="'+model.locked.includes(i)+'" aria-label="'+label+'"></span>').join('')+'</div>';
range.innerHTML='<div data-model-reasoning-effort-slider><div data-orientation="horizontal" aria-disabled="false" style="width:230px;height:24px"><span role="slider" tabindex="0" aria-valuemin="'+model.min+'" aria-valuemax="'+(model.min+model.labels.length-1)+'" aria-valuenow="'+model.value+'"></span></div>'+model.labels.map((label,i)=>'<span data-selected="'+(i===index)+'" data-locked="'+model.locked.includes(i)+'" aria-label="'+label+'"></span>').join('')+'</div>';
document.getElementById('effort-description').textContent=document.body.hasAttribute('data-break-effort-announcement')?'Unknown effort':family+' '+model.labels[index]+', '+(index+1)+' of '+model.labels.length+'.';
control.textContent=model.labels[index];
document.querySelectorAll('[data-family]').forEach(row=>row.setAttribute('aria-checked',String(row.dataset.family===family)));
}
control.onclick=()=>{picker.hidden=false;control.setAttribute('aria-expanded','true');};
document.querySelectorAll('[data-family]').forEach(row=>row.onclick=()=>{family=row.dataset.family;render();});
range.onkeydown=event=>{
if (!['ArrowLeft','ArrowRight'].includes(event.key)) return;
event.preventDefault();const model=definitions[family], step=event.key==='ArrowRight'?1:-1;
range.onkeydown=event=>{
if (!['ArrowLeft','ArrowRight'].includes(event.key)) return;
if (document.body.dataset.pointerOnly==='true') return;
if (document.body.dataset.sliderOnly==='true' && event.target.getAttribute('role')!=='slider') return;
document.body.dataset.sliderKeys=String(+(document.body.dataset.sliderKeys||0)+1);
event.preventDefault();const model=definitions[family], step=event.key==='ArrowRight'?1:-1;
let next=model.value+step;
while(next>=model.min&&next<model.min+model.labels.length&&model.locked.includes(next-model.min)) next+=step;
if(next>=model.min&&next<model.min+model.labels.length) model.value=next;
render();
};
render();
};
range.onpointerdown=event=>{
if (document.body.dataset.pointerOnly!=='true') return;
const track=event.target.closest('[data-orientation="horizontal"]');
if (!track) return;
const rect=track.getBoundingClientRect(), model=definitions[family];
const margin=14, ratio=Math.max(0,Math.min(1,(event.clientX-rect.left-margin)/(rect.width-2*margin)));
const index=Math.round(ratio*(model.labels.length-1));
if (model.locked.includes(index)) return;
document.body.dataset.pointerSteps=String(+(document.body.dataset.pointerSteps||0)+1);
model.value=model.min+index;render();
};
document.addEventListener('keydown',event=>{if(event.key==='Escape'){picker.hidden=true;control.setAttribute('aria-expanded','false');}});
render();
</script>`;
Expand All @@ -50,7 +64,8 @@ test.skipIf(process.env.CODEXIFY_TEST_BROWSER_COMPANION_CHROME !== "1")(
"Chrome discovers each family's native effort range, labels and locks and restores the original selection",
async () => {
const profilePath = await mkdtemp(join(tmpdir(), "native-effort-discovery-"));
const executablePath = await privateChromeExecutable(join(import.meta.dirname, "../.browsers", `${process.platform}-${process.arch}`));
const executablePath = process.env.CODEXIFY_TEST_BROWSER_COMPANION_CHROME_PATH
?? await privateChromeExecutable(join(import.meta.dirname, "../.browsers", `${process.platform}-${process.arch}`));
const context = await chromium.launchPersistentContext(profilePath, {channel:"chrome",executablePath,headless:true,viewport:null});
try {
await context.route("https://chatgpt.com/**", route => route.fulfill({contentType:"text/html",body:fixture}));
Expand Down Expand Up @@ -86,3 +101,32 @@ test.skipIf(process.env.CODEXIFY_TEST_BROWSER_COMPANION_CHROME !== "1")(
} finally { await context.close(); await rm(profilePath, {recursive:true,force:true}); }
}, 30_000,
);

for (const mode of ["sliderOnly", "pointerOnly"] as const) {
test.skipIf(process.env.CODEXIFY_TEST_BROWSER_COMPANION_CHROME !== "1")(
`Chrome effort discovery works with ${mode} interaction and restores selection`,
async () => {
const profilePath = await mkdtemp(join(tmpdir(), `native-effort-${mode}-`));
const executablePath = process.env.CODEXIFY_TEST_BROWSER_COMPANION_CHROME_PATH
?? await privateChromeExecutable(join(import.meta.dirname, "../.browsers", `${process.platform}-${process.arch}`));
const context = await chromium.launchPersistentContext(profilePath, {channel:"chrome",executablePath,headless:true,viewport:null});
try {
await context.route("https://chatgpt.com/**", route => route.fulfill({contentType:"text/html",body:fixture}));
const page = context.pages()[0] ?? await context.newPage();
await page.goto("https://chatgpt.com/");
await page.evaluate(mode => { document.body.dataset[mode] = "true"; }, mode);
await page.locator("#prompt-textarea").focus();
const capabilities = await detectChatGptAccountCapabilities(page);
expect(capabilities.models.some(model => model.family === "6")).toBe(true);
expect(await page.locator('[role="slider"]').getAttribute("aria-valuenow")).toBe("2");
expect(await page.locator("#effort").getAttribute("aria-expanded")).toBe("false");
const count = await page.evaluate(mode => Number(document.body.dataset[mode === "sliderOnly" ? "sliderKeys" : "pointerSteps"] || "0"), mode);
expect(count).toBeGreaterThan(0);
} finally {
await context.close();
await rm(profilePath, {recursive:true,force:true});
}
},
60_000,
);
}
Loading