From b536bf57faf5c5e4dc563da026e0380ca27f5f7e Mon Sep 17 00:00:00 2001
From: Soham Kukreti <kukretisoham@gmail.com>
Date: Thu, 13 Aug 2026 20:47:03 +0530
Subject: [PATCH] fix: remove unconditional setTimeout waits from
 overlay/consent removal scripts

In-page timers never fire on CSP-sandboxed pages (GitHub/HuggingFace raw), hanging crawls 30s-to-forever; waits now run Python-side or only after a consent action actually fired.
---
 crawl4ai/async_crawler_strategy.py            |  2 +-
 crawl4ai/js_snippet/remove_consent_popups.js  |  9 +++++++--
 .../js_snippet/remove_overlay_elements.js     |  2 --
 tests/regression/test_reg_browser.py          | 20 +++++++++++++++++++
 4 files changed, 28 insertions(+), 5 deletions(-)

diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py
index 265c376e9..843463aa6 100644
--- a/crawl4ai/async_crawler_strategy.py
+++ b/crawl4ai/async_crawler_strategy.py
@@ -1534,7 +1534,7 @@ async def remove_overlay_elements(self, page: Page) -> None:
                 }})()
             """
             )
-            await page.wait_for_timeout(500)  # Wait for any animations to complete
+            await page.wait_for_timeout(600)  # Wait for any animations to complete
         except Exception as e:
             self.logger.warning(
                 message="Failed to remove overlay elements: {error}",
diff --git a/crawl4ai/js_snippet/remove_consent_popups.js b/crawl4ai/js_snippet/remove_consent_popups.js
index 9aac8d345..2f7eb0f34 100644
--- a/crawl4ai/js_snippet/remove_consent_popups.js
+++ b/crawl4ai/js_snippet/remove_consent_popups.js
@@ -292,6 +292,7 @@ async () => {
     // =========================================================================
     // Phase 2: Try CMP JavaScript APIs
     // =========================================================================
+    let apiCalled = false;
 
     // IAB TCF v2 API
     if (typeof window.__tcfapi === 'function') {
@@ -304,6 +305,7 @@ async () => {
     if (typeof window.Didomi !== 'undefined') {
         try {
             window.Didomi.setUserAgreeToAll();
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
@@ -311,6 +313,7 @@ async () => {
     if (typeof window.Cookiebot !== 'undefined') {
         try {
             window.Cookiebot.submitCustomConsent(true, true, true);
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
@@ -318,6 +321,7 @@ async () => {
     if (typeof window.Osano !== 'undefined') {
         try {
             window.Osano.cm.acceptAll();
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
@@ -325,11 +329,12 @@ async () => {
     if (typeof window.klaro !== 'undefined') {
         try {
             window.klaro.getManager().acceptAll();
+            apiCalled = true;
         } catch (e) { /* continue */ }
     }
 
-    // Wait for CMP animations/transitions
-    await new Promise(r => setTimeout(r, 500));
+    // Wait for CMP animations/transitions - only when a consent action actually fired
+    if (accepted || apiCalled) await new Promise(r => setTimeout(r, 500));
 
     // =========================================================================
     // Phase 3: Remove known CMP containers by selector
diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.js
index a50d94274..36de9d971 100644
--- a/crawl4ai/js_snippet/remove_overlay_elements.js
+++ b/crawl4ai/js_snippet/remove_overlay_elements.js
@@ -114,7 +114,5 @@ async () => {
     document.body.style.paddingRight = "0px";
     document.body.style.overflow = "auto";
 
-    // Wait a bit for any animations to complete
     document.body.scrollIntoView(false);
-    await new Promise((resolve) => setTimeout(resolve, 50));
 };
diff --git a/tests/regression/test_reg_browser.py b/tests/regression/test_reg_browser.py
index ba901178b..2d9f87fb4 100644
--- a/tests/regression/test_reg_browser.py
+++ b/tests/regression/test_reg_browser.py
@@ -289,6 +289,26 @@ async def test_remove_overlay_elements(local_server):
         assert len(result.html) > 0, "HTML should still be present after overlay removal"
 
 
+@pytest.mark.asyncio
+@pytest.mark.network
+async def test_overlay_removal_on_csp_sandbox_page():
+    """raw.githubusercontent.com serves CSP `sandbox`, which disables page
+    timers; overlay/consent removal must not stall waiting on in-page
+    setTimeout (used to hang ~30s per page). URL is commit-pinned so the
+    response never changes."""
+    url = "https://raw.githubusercontent.com/unclecode/crawl4ai/055e2ecdc702228a80363e2c51ca79e473222072/README.md"
+    config = CrawlerRunConfig(
+        remove_overlay_elements=True, remove_consent_popups=True, verbose=False
+    )
+    async with AsyncWebCrawler(config=BrowserConfig(headless=True, verbose=False)) as crawler:
+        start = time.perf_counter()
+        result = await crawler.arun(url=url, config=config)
+        elapsed = time.perf_counter() - start
+        assert result.success, f"Crawl failed on CSP-sandboxed page: {result.error_message}"
+        assert "Crawl4AI" in result.html, "Page content should be captured"
+        assert elapsed < 20, f"Overlay removal stalled on CSP-sandboxed page ({elapsed:.1f}s)"
+
+
 # ---------------------------------------------------------------------------
 # Stealth mode
 # ---------------------------------------------------------------------------