Remove unconditional setTimeout waits from overlay/consent removal scripts

crawl4ai/async_crawler_strategy.py

@@ -1534,7 +1534,7 @@ async def remove_overlay_elements(self, page: Page) -> None:
                 }})()
             """
             )
-            await page.wait_for_timeout(500)  # Wait for any animations to complete
+            await page.wait_for_timeout(600)  # Wait for any animations to complete
         except Exception as e:
             self.logger.warning(
                 message="Failed to remove overlay elements: {error}",

crawl4ai/js_snippet/remove_consent_popups.js

@@ -292,6 +292,7 @@ async () => {
     // =========================================================================
     // Phase 2: Try CMP JavaScript APIs
     // =========================================================================
+    let apiCalled = false;
 
     // IAB TCF v2 API
     if (typeof window.__tcfapi === 'function') {
@@ -304,32 +305,36 @@ async () => {
     if (typeof window.Didomi !== 'undefined') {
         try {
             window.Didomi.setUserAgreeToAll();
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
     // Cookiebot API
     if (typeof window.Cookiebot !== 'undefined') {
         try {
             window.Cookiebot.submitCustomConsent(true, true, true);
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
     // Osano API
     if (typeof window.Osano !== 'undefined') {
         try {
             window.Osano.cm.acceptAll();
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
     // Klaro API
     if (typeof window.klaro !== 'undefined') {
         try {
             window.klaro.getManager().acceptAll();
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
-    // Wait for CMP animations/transitions
-    await new Promise(r => setTimeout(r, 500));
+    // Wait for CMP animations/transitions - only when a consent action actually fired
+    if (accepted || apiCalled) await new Promise(r => setTimeout(r, 500));
 
     // =========================================================================
     // Phase 3: Remove known CMP containers by selector

crawl4ai/js_snippet/remove_overlay_elements.js

@@ -114,7 +114,5 @@ async () => {
     document.body.style.paddingRight = "0px";
     document.body.style.overflow = "auto";
 
-    // Wait a bit for any animations to complete
     document.body.scrollIntoView(false);
-    await new Promise((resolve) => setTimeout(resolve, 50));
 };

tests/regression/test_reg_browser.py

@@ -289,6 +289,26 @@ async def test_remove_overlay_elements(local_server):
         assert len(result.html) > 0, "HTML should still be present after overlay removal"
 
 
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_overlay_removal_on_csp_sandbox_page():
+    """raw.githubusercontent.com serves CSP `sandbox`, which disables page
+    timers; overlay/consent removal must not stall waiting on in-page
+    setTimeout (used to hang ~30s per page). URL is commit-pinned so the
+    response never changes."""
+    url = "https://raw.githubusercontent.com/unclecode/crawl4ai/055e2ecdc702228a80363e2c51ca79e473222072/README.md"
+    config = CrawlerRunConfig(
+        remove_overlay_elements=True, remove_consent_popups=True, verbose=False
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        start = time.perf_counter()
+        result = await crawler.arun(url=url, config=config)
+        elapsed = time.perf_counter() - start
+        assert result.success, f"Crawl failed on CSP-sandboxed page: {result.error_message}"
+        assert "Crawl4AI" in result.html, "Page content should be captured"
+        assert elapsed < 20, f"Overlay removal stalled on CSP-sandboxed page ({elapsed:.1f}s)"
+
+
 # ---------------------------------------------------------------------------
 # Stealth mode
 # ---------------------------------------------------------------------------