Skip to content

Commit e2ccfa7

Browse files
author
stlc-bot
committed
fix(scrape): preserve successful formats when other outputs fail (#1263)
Stainless-Generated-From: 85adbf2
1 parent 492a305 commit e2ccfa7

7 files changed

Lines changed: 467 additions & 162 deletions

File tree

‎lib/context_dev/models/web_scrape_params.rb‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -89,12 +89,16 @@ class WebScrapeParams < ContextDev::Internal::Type::BaseModel
8989

9090
# @!attribute timeout_opts
9191
# Total deadline, including navigation, actions, waiting, and all outputs.
92-
# Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
93-
# the current page state and return captured images if image processing cannot
94-
# finish before the deadline; these responses set isPartial and are not cached.
95-
# Every requested format must still be available. Fixed waits must fit before a
96-
# response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
97-
# when using return-partial.
92+
# Defaults to 60000 milliseconds with behavior fail. Individual outputs have
93+
# internal deadlines that reserve time to return completed outputs; timed-out
94+
# outputs have success: false and data: null under either behavior. The overall
95+
# request deadline remains enforced: fail returns an error if that deadline is
96+
# reached. Use return-partial to allow the current page state and available
97+
# outputs when the page is still loading. Partial responses set isPartial. Failed
98+
# retrievals and incomplete captures are not cached; valid captured pieces may be
99+
# cached independently. Fixed waits must fit before a response reserve of up to
100+
# 5000 milliseconds (at most one quarter of the timeout) when using
101+
# return-partial.
98102
#
99103
# @return [ContextDev::Models::WebScrapeParams::TimeoutOpts, nil]
100104
optional :timeout_opts, -> { ContextDev::WebScrapeParams::TimeoutOpts }, api_name: :timeoutOpts
@@ -149,7 +153,7 @@ class Formats < ContextDev::Internal::Type::BaseModel
149153

150154
# @!attribute highlights
151155
# Relevant passages for your question or topic, with headings included when needed
152-
# for context. Adds 3 credits.
156+
# for context. Adds 3 credits when passages are returned.
153157
#
154158
# @return [Boolean, nil]
155159
optional :highlights, ContextDev::Internal::Type::Boolean
@@ -167,7 +171,8 @@ class Formats < ContextDev::Internal::Type::BaseModel
167171
optional :images, ContextDev::Internal::Type::Boolean
168172

169173
# @!attribute json
170-
# Page data extracted using your schema. Adds 4 credits.
174+
# Page data extracted using your schema. Adds 4 credits when extraction succeeds
175+
# and its result is returned.
171176
#
172177
# @return [Boolean, nil]
173178
optional :json, ContextDev::Internal::Type::Boolean
@@ -185,7 +190,8 @@ class Formats < ContextDev::Internal::Type::BaseModel
185190
optional :parse, ContextDev::Internal::Type::Boolean
186191

187192
# @!attribute product
188-
# Product details such as name, price, and availability. Adds 1 credit.
193+
# Product details such as name, price, and availability. Adds 1 credit when its
194+
# successful result is returned or the target page is missing.
189195
#
190196
# @return [Boolean, nil]
191197
optional :product, ContextDev::Internal::Type::Boolean
@@ -210,13 +216,13 @@ class Formats < ContextDev::Internal::Type::BaseModel
210216
#
211217
# @param images [Boolean] Images found on the page.
212218
#
213-
# @param json [Boolean] Page data extracted using your schema. Adds 4 credits.
219+
# @param json [Boolean] Page data extracted using your schema. Adds 4 credits when extraction succeeds a
214220
#
215221
# @param markdown [Boolean] Page content as Markdown.
216222
#
217223
# @param parse [Boolean] Fields selected by parseParams.rules.
218224
#
219-
# @param product [Boolean] Product details such as name, price, and availability. Adds 1 credit.
225+
# @param product [Boolean] Product details such as name, price, and availability. Adds 1 credit when its su
220226
#
221227
# @param screenshot [Boolean] An inline image of the page.
222228
end
@@ -462,9 +468,10 @@ module Type
462468
class ProductParams < ContextDev::Internal::Type::BaseModel
463469
# @!attribute use_ai_fallback
464470
# Extract the product with a specialized model when the page has no structured
465-
# product data. Adds six credits when the model returns a verdict. If the fallback
466-
# fails, returns a partial response with the deterministic result and no fallback
467-
# charge. Request deadlines and client disconnects still apply.
471+
# product data. Adds six credits when the model verdict is returned successfully.
472+
# If the fallback fails, the product output has success: false and data: null with
473+
# no fallback charge; other outputs remain available. Request deadlines and client
474+
# disconnects still apply.
468475
#
469476
# @return [Boolean, nil]
470477
optional :use_ai_fallback, ContextDev::Internal::Type::Boolean, api_name: :useAIFallback
@@ -984,12 +991,16 @@ class TimeoutOpts < ContextDev::Internal::Type::BaseModel
984991
# {ContextDev::Models::WebScrapeParams::TimeoutOpts} for more details.
985992
#
986993
# Total deadline, including navigation, actions, waiting, and all outputs.
987-
# Defaults to 60000 milliseconds with behavior fail. Use return-partial to capture
988-
# the current page state and return captured images if image processing cannot
989-
# finish before the deadline; these responses set isPartial and are not cached.
990-
# Every requested format must still be available. Fixed waits must fit before a
991-
# response reserve of up to 5000 milliseconds (at most one quarter of the timeout)
992-
# when using return-partial.
994+
# Defaults to 60000 milliseconds with behavior fail. Individual outputs have
995+
# internal deadlines that reserve time to return completed outputs; timed-out
996+
# outputs have success: false and data: null under either behavior. The overall
997+
# request deadline remains enforced: fail returns an error if that deadline is
998+
# reached. Use return-partial to allow the current page state and available
999+
# outputs when the page is still loading. Partial responses set isPartial. Failed
1000+
# retrievals and incomplete captures are not cached; valid captured pieces may be
1001+
# cached independently. Fixed waits must fit before a response reserve of up to
1002+
# 5000 milliseconds (at most one quarter of the timeout) when using
1003+
# return-partial.
9931004
#
9941005
# @param milliseconds [Integer] Request deadline in milliseconds. Maximum: 300000 (5 minutes).
9951006
#

‎lib/context_dev/models/web_scrape_response.rb‎

Lines changed: 101 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -89,10 +89,11 @@ class WebScrapeResponse < ContextDev::Internal::Type::BaseModel
8989
required :url, String
9090

9191
# @!attribute is_partial
92-
# Present when return-partial captures a page that is still loading, returns
93-
# images before image processing finishes, or cuts product AI extraction short.
94-
# Also present if the optional product AI fallback fails. Partial responses are
95-
# not cached.
92+
# Present when a requested output fails, capture returns a page that is still
93+
# loading, images return before processing finishes, or the optional product AI
94+
# fallback fails or is cut short. Check each output's success field for its
95+
# result. Valid captured pieces may be cached independently; failed retrievals and
96+
# incomplete captures are not cached.
9697
#
9798
# @return [Boolean, ContextDev::Models::WebScrapeResponse::IsPartial, nil]
9899
optional :is_partial, enum: -> { ContextDev::Models::WebScrapeResponse::IsPartial }, api_name: :isPartial
@@ -133,7 +134,7 @@ class WebScrapeResponse < ContextDev::Internal::Type::BaseModel
133134
#
134135
# @param url [String] Final URL after redirects and browser actions.
135136
#
136-
# @param is_partial [Boolean, ContextDev::Models::WebScrapeResponse::IsPartial] Present when return-partial captures a page that is still loading, returns image
137+
# @param is_partial [Boolean, ContextDev::Models::WebScrapeResponse::IsPartial] Present when a requested output fails, capture returns a page that is still load
137138
#
138139
# @param key_metadata [ContextDev::Models::WebScrapeResponse::KeyMetadata] Credit usage, included whenever a valid API key is provided.
139140

@@ -149,12 +150,21 @@ class Bytes < ContextDev::Internal::Type::BaseModel
149150
# @return [Boolean]
150151
required :requested, ContextDev::Internal::Type::Boolean
151152

152-
# @!method initialize(data:, requested:)
153+
# @!attribute success
154+
# True when retrieved, false when retrieval failed, and null when not requested.
155+
#
156+
# @return [Boolean, nil]
157+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
158+
159+
# @!method initialize(data:, requested:, success:)
153160
# Original HTTP response body. Waiting, actions, and content filters never change
154161
# it.
155162
#
156163
# @param data [ContextDev::Models::WebScrapeResponse::Bytes::Data, nil]
164+
#
157165
# @param requested [Boolean]
166+
#
167+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
158168

159169
# @see ContextDev::Models::WebScrapeResponse::Bytes#data
160170
class Data < ContextDev::Internal::Type::BaseModel
@@ -235,13 +245,22 @@ class Highlights < ContextDev::Internal::Type::BaseModel
235245
# @return [Boolean]
236246
required :requested, ContextDev::Internal::Type::Boolean
237247

238-
# @!method initialize(data:, requested:)
248+
# @!attribute success
249+
# True when retrieved, false when retrieval failed, and null when not requested.
250+
#
251+
# @return [Boolean, nil]
252+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
253+
254+
# @!method initialize(data:, requested:, success:)
239255
# Relevant passages for your question or topic, in page order. A heading in square
240256
# brackets is included when needed to interpret a passage. Empty when the page has
241257
# no text.
242258
#
243259
# @param data [Array<String>, nil]
260+
#
244261
# @param requested [Boolean]
262+
#
263+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
245264
end
246265

247266
# @see ContextDev::Models::WebScrapeResponse#html
@@ -256,11 +275,20 @@ class HTML < ContextDev::Internal::Type::BaseModel
256275
# @return [Boolean]
257276
required :requested, ContextDev::Internal::Type::Boolean
258277

259-
# @!method initialize(data:, requested:)
278+
# @!attribute success
279+
# True when retrieved, false when retrieval failed, and null when not requested.
280+
#
281+
# @return [Boolean, nil]
282+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
283+
284+
# @!method initialize(data:, requested:, success:)
260285
# Rendered HTML after content filters.
261286
#
262287
# @param data [String, nil]
288+
#
263289
# @param requested [Boolean]
290+
#
291+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
264292
end
265293

266294
# @see ContextDev::Models::WebScrapeResponse#images
@@ -277,11 +305,20 @@ class Images < ContextDev::Internal::Type::BaseModel
277305
# @return [Boolean]
278306
required :requested, ContextDev::Internal::Type::Boolean
279307

280-
# @!method initialize(data:, requested:)
308+
# @!attribute success
309+
# True when retrieved, false when retrieval failed, and null when not requested.
310+
#
311+
# @return [Boolean, nil]
312+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
313+
314+
# @!method initialize(data:, requested:, success:)
281315
# Images after content filters. Empty when none are found.
282316
#
283317
# @param data [Array<ContextDev::Models::WebScrapeResponse::Images::Data>, nil]
318+
#
284319
# @param requested [Boolean]
320+
#
321+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
285322

286323
class Data < ContextDev::Internal::Type::BaseModel
287324
# @!attribute alt
@@ -365,11 +402,20 @@ class Json < ContextDev::Internal::Type::BaseModel
365402
# @return [Boolean]
366403
required :requested, ContextDev::Internal::Type::Boolean
367404

368-
# @!method initialize(data:, requested:)
405+
# @!attribute success
406+
# True when retrieved, false when retrieval failed, and null when not requested.
407+
#
408+
# @return [Boolean, nil]
409+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
410+
411+
# @!method initialize(data:, requested:, success:)
369412
# Page data extracted using your schema.
370413
#
371414
# @param data [Hash{Symbol=>Object}, nil]
415+
#
372416
# @param requested [Boolean]
417+
#
418+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
373419
end
374420

375421
# @see ContextDev::Models::WebScrapeResponse#markdown
@@ -384,11 +430,20 @@ class Markdown < ContextDev::Internal::Type::BaseModel
384430
# @return [Boolean]
385431
required :requested, ContextDev::Internal::Type::Boolean
386432

387-
# @!method initialize(data:, requested:)
433+
# @!attribute success
434+
# True when retrieved, false when retrieval failed, and null when not requested.
435+
#
436+
# @return [Boolean, nil]
437+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
438+
439+
# @!method initialize(data:, requested:, success:)
388440
# Markdown after content filters.
389441
#
390442
# @param data [String, nil]
443+
#
391444
# @param requested [Boolean]
445+
#
446+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
392447
end
393448

394449
# @see ContextDev::Models::WebScrapeResponse#metadata
@@ -662,11 +717,20 @@ class Parsed < ContextDev::Internal::Type::BaseModel
662717
# @return [Boolean]
663718
required :requested, ContextDev::Internal::Type::Boolean
664719

665-
# @!method initialize(data:, requested:)
720+
# @!attribute success
721+
# True when retrieved, false when retrieval failed, and null when not requested.
722+
#
723+
# @return [Boolean, nil]
724+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
725+
726+
# @!method initialize(data:, requested:, success:)
666727
# Fields produced by parseParams.rules, after shared content filters.
667728
#
668729
# @param data [Hash{Symbol=>Object}, nil]
730+
#
669731
# @param requested [Boolean]
732+
#
733+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
670734
end
671735

672736
# @see ContextDev::Models::WebScrapeResponse#product
@@ -681,11 +745,20 @@ class Product < ContextDev::Internal::Type::BaseModel
681745
# @return [Boolean]
682746
required :requested, ContextDev::Internal::Type::Boolean
683747

684-
# @!method initialize(data:, requested:)
748+
# @!attribute success
749+
# True when retrieved, false when retrieval failed, and null when not requested.
750+
#
751+
# @return [Boolean, nil]
752+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
753+
754+
# @!method initialize(data:, requested:, success:)
685755
# Product details found on the page.
686756
#
687757
# @param data [ContextDev::Models::WebScrapeResponse::Product::Data, nil]
758+
#
688759
# @param requested [Boolean]
760+
#
761+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
689762

690763
# @see ContextDev::Models::WebScrapeResponse::Product#data
691764
class Data < ContextDev::Internal::Type::BaseModel
@@ -920,17 +993,27 @@ class Screenshot < ContextDev::Internal::Type::BaseModel
920993
# @return [Boolean]
921994
required :requested, ContextDev::Internal::Type::Boolean
922995

923-
# @!method initialize(data:, requested:)
996+
# @!attribute success
997+
# True when retrieved, false when retrieval failed, and null when not requested.
998+
#
999+
# @return [Boolean, nil]
1000+
required :success, ContextDev::Internal::Type::Boolean, nil?: true
1001+
1002+
# @!method initialize(data:, requested:, success:)
9241003
# An image data URL. Use directly as an image src.
9251004
#
9261005
# @param data [String, nil]
1006+
#
9271007
# @param requested [Boolean]
1008+
#
1009+
# @param success [Boolean, nil] True when retrieved, false when retrieval failed, and null when not requested.
9281010
end
9291011

930-
# Present when return-partial captures a page that is still loading, returns
931-
# images before image processing finishes, or cuts product AI extraction short.
932-
# Also present if the optional product AI fallback fails. Partial responses are
933-
# not cached.
1012+
# Present when a requested output fails, capture returns a page that is still
1013+
# loading, images return before processing finishes, or the optional product AI
1014+
# fallback fails or is cut short. Check each output's success field for its
1015+
# result. Valid captured pieces may be cached independently; failed retrievals and
1016+
# incomplete captures are not cached.
9341017
#
9351018
# @see ContextDev::Models::WebScrapeResponse#is_partial
9361019
module IsPartial

‎lib/context_dev/resources/web.rb‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -187,14 +187,20 @@ def map_urls(params)
187187
# extraction. Cached outputs can come from different visits within maxAgeMs; use 0
188188
# for a fresh capture. HTML-only requests use the existing fast acquisition path.
189189
# Highlights return the plain-text passages most relevant to
190-
# highlightsParams.query. One credit per request, including cache hits and missing
191-
# pages, or two with browser actions; highlights add 3 credits when passages are
192-
# returned; JSON extraction adds four credits and runs an LLM over the page
193-
# Markdown on every request that has text to extract; PDF OCR adds one credit per
194-
# recovered page on fresh extraction; the product output adds one credit, plus six
195-
# more when the specialized model is used. Original response bytes and screenshots
196-
# are limited to 20 MiB each, screenshots to 40 megapixels, and the combined
197-
# browser capture to 60 MiB.
190+
# highlightsParams.query. Requests with at least one successful output cost one
191+
# base credit, including cache hits, or two with browser actions. All-failed
192+
# responses are unbilled except missing pages, which retain the base price and the
193+
# one-credit product charge when product was requested. Highlights add 3 credits
194+
# when passages are returned. JSON extraction runs an LLM over nonempty page
195+
# Markdown and adds four credits only when its result is returned successfully.
196+
# PDF OCR adds one credit per recovered page on fresh extraction. Product adds one
197+
# credit when its successful result is returned, plus six if that result used the
198+
# specialized model. Original response bytes and screenshots are limited to 20 MiB
199+
# each, screenshots to 40 megapixels, and the combined response to 60 MiB. An
200+
# oversized output has success: false and data: null. If the combined response
201+
# exceeds its limit, the largest outputs are marked failed until the remaining
202+
# outputs fit. Valid captured pieces may still be cached when omitted to meet the
203+
# response size limit.
198204
#
199205
# @overload scrape(formats:, url:, highlights_params: nil, image_params: nil, json_params: nil, markdown_params: nil, max_age_ms: nil, parse_params: nil, product_params: nil, screenshot_params: nil, shared_params: nil, tags: nil, timeout_opts: nil, zdr: nil, request_options: {})
200206
#

0 commit comments

Comments
 (0)