diff --git a/.release-please-manifest.json b/.release-please-manifest.json index bfaab56f..0c0c0c35 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.24.0" + ".": "1.25.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index af822916..4eb43d23 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 23 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-079e4949baaeab93254aa8a651f4e9976e95387b1633002e925114ce36f29013.yml -openapi_spec_hash: 331f27b62376aaf9c311af03a21d31b8 -config_hash: ada0970327c433e075b4ceef583e0755 +configured_endpoints: 24 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-cc8ca2c80e73bdcff491c96d9289c9e88d831215375ef83632cc51f8e8e06183.yml +openapi_spec_hash: bf55dce481efcf2f24bc9ff59c2e3f61 +config_hash: cf2c7f48b329e0a61612b30cd2755ad1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 852c36eb..07132f42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 1.25.0 (2026-05-31) + +Full Changelog: [v1.24.0...v1.25.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.24.0...v1.25.0) + +### Features + +* **api:** manual updates ([688e5f5](https://github.com/context-dot-dev/context-ruby-sdk/commit/688e5f56137aeac7c5219383f241a459173f961f)) + ## 1.24.0 (2026-05-30) Full Changelog: [v1.23.0...v1.24.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.23.0...v1.24.0) diff --git a/Gemfile.lock b/Gemfile.lock index 6422e86a..1fd394c1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT PATH remote: . specs: - context.dev (1.24.0) + context.dev (1.25.0) cgi connection_pool diff --git a/README.md b/README.md index b52d19fe..5becb3ed 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ To use this gem, install via Bundler by adding the following to your application ```ruby -gem "context.dev", "~> 1.24.0" +gem "context.dev", "~> 1.25.0" ``` diff --git a/lib/context_dev.rb b/lib/context_dev.rb index 84c37a57..cc510200 100644 --- a/lib/context_dev.rb +++ b/lib/context_dev.rb @@ -82,6 +82,8 @@ require_relative "context_dev/models/utility_prefetch_response" require_relative "context_dev/models/web_extract_fonts_params" require_relative "context_dev/models/web_extract_fonts_response" +require_relative "context_dev/models/web_extract_params" +require_relative "context_dev/models/web_extract_response" require_relative "context_dev/models/web_extract_styleguide_params" require_relative "context_dev/models/web_extract_styleguide_response" require_relative "context_dev/models/web_screenshot_params" diff --git a/lib/context_dev/internal/type/base_model.rb b/lib/context_dev/internal/type/base_model.rb index 41f5ee16..af0e1db9 100644 --- a/lib/context_dev/internal/type/base_model.rb +++ b/lib/context_dev/internal/type/base_model.rb @@ -438,11 +438,11 @@ def deep_to_h = self.class.recursively_to_h(@data, convert: false) # @return [Hash{Symbol=>Object}] # # @example - # # `web_extract_fonts_response` is a `ContextDev::Models::WebExtractFontsResponse` - # web_extract_fonts_response => { - # code: code, - # domain: domain, - # fonts: fonts + # # `web_extract_response` is a `ContextDev::Models::WebExtractResponse` + # web_extract_response => { + # data: data, + # metadata: metadata, + # status: status # } def deconstruct_keys(keys) (keys || self.class.known_fields.keys) diff --git a/lib/context_dev/models.rb b/lib/context_dev/models.rb index 7117abd9..601e3b55 100644 --- a/lib/context_dev/models.rb +++ b/lib/context_dev/models.rb @@ -69,6 +69,8 @@ module ContextDev WebExtractFontsParams = ContextDev::Models::WebExtractFontsParams + WebExtractParams = ContextDev::Models::WebExtractParams + WebExtractStyleguideParams = ContextDev::Models::WebExtractStyleguideParams WebScreenshotParams = ContextDev::Models::WebScreenshotParams diff --git a/lib/context_dev/models/web_extract_params.rb b/lib/context_dev/models/web_extract_params.rb new file mode 100644 index 00000000..641f2f8c --- /dev/null +++ b/lib/context_dev/models/web_extract_params.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +module ContextDev + module Models + # @see ContextDev::Resources::Web#extract + class WebExtractParams < ContextDev::Internal::Type::BaseModel + extend ContextDev::Internal::Type::RequestParameters::Converter + include ContextDev::Internal::Type::RequestParameters + + # @!attribute schema + # JSON Schema for the returned data object. TypeScript Zod users can pass a JSON + # Schema generated from a Zod object; Python users can pass the equivalent JSON + # Schema object. + # + # @return [Hash{Symbol=>Object}] + required :schema, ContextDev::Internal::Type::HashOf[ContextDev::Internal::Type::Unknown] + + # @!attribute url + # The starting website URL to crawl and extract from. Must include http:// or + # https://. + # + # @return [String] + required :url, String + + # @!attribute fact_check + # When true (default), every returned value must be grounded in facts stated on + # the page; fields that cannot be supported by the page are returned as + # null/empty. When false, the model may make reasonable inferences and derivations + # from the page content (e.g. ideal customer, competitor analysis, + # recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, + # metrics) faithful to the source. + # + # @return [Boolean, nil] + optional :fact_check, ContextDev::Internal::Type::Boolean, api_name: :factCheck + + # @!attribute follow_subdomains + # When true, follow links on subdomains of the starting URL's domain. + # + # @return [Boolean, nil] + optional :follow_subdomains, ContextDev::Internal::Type::Boolean, api_name: :followSubdomains + + # @!attribute include_frames + # When true, iframe contents are included in Markdown before extraction. + # + # @return [Boolean, nil] + optional :include_frames, ContextDev::Internal::Type::Boolean, api_name: :includeFrames + + # @!attribute instructions + # Optional extraction guidance, such as which facts to prioritize or how to + # interpret fields in the schema. + # + # @return [String, nil] + optional :instructions, String + + # @!attribute max_age_ms + # Return cached scrape results if a prior scrape for the same parameters is + # younger than this many milliseconds. + # + # @return [Integer, nil] + optional :max_age_ms, Integer, api_name: :maxAgeMs + + # @!attribute pdf + # + # @return [ContextDev::Models::WebExtractParams::Pdf, nil] + optional :pdf, -> { ContextDev::WebExtractParams::Pdf } + + # @!attribute stop_after_ms + # Soft time budget for the crawl in milliseconds. + # + # @return [Integer, nil] + optional :stop_after_ms, Integer, api_name: :stopAfterMs + + # @!attribute timeout_ms + # Optional timeout in milliseconds for the request. If the request takes longer + # than this value, it will be aborted with a 408 status code. Maximum allowed + # value is 300000ms (5 minutes). + # + # @return [Integer, nil] + optional :timeout_ms, Integer, api_name: :timeoutMS + + # @!attribute wait_for_ms + # Optional browser wait time in milliseconds after initial page load for each + # crawled page. + # + # @return [Integer, nil] + optional :wait_for_ms, Integer, api_name: :waitForMs + + # @!method initialize(schema:, url:, fact_check: nil, follow_subdomains: nil, include_frames: nil, instructions: nil, max_age_ms: nil, pdf: nil, stop_after_ms: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {}) + # Some parameter documentations has been truncated, see + # {ContextDev::Models::WebExtractParams} for more details. + # + # @param schema [Hash{Symbol=>Object}] JSON Schema for the returned data object. TypeScript Zod users can pass a JSON S + # + # @param url [String] The starting website URL to crawl and extract from. Must include http:// or http + # + # @param fact_check [Boolean] When true (default), every returned value must be grounded in facts stated on th + # + # @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain. + # + # @param include_frames [Boolean] When true, iframe contents are included in Markdown before extraction. + # + # @param instructions [String] Optional extraction guidance, such as which facts to prioritize or how to interp + # + # @param max_age_ms [Integer] Return cached scrape results if a prior scrape for the same parameters is younge + # + # @param pdf [ContextDev::Models::WebExtractParams::Pdf] + # + # @param stop_after_ms [Integer] Soft time budget for the crawl in milliseconds. + # + # @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th + # + # @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load for each craw + # + # @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}] + + class Pdf < ContextDev::Internal::Type::BaseModel + # @!attribute end_ + # Last 1-based PDF page to parse. Must be greater than or equal to start when both + # are provided. + # + # @return [Integer, nil] + optional :end_, Integer, api_name: :end + + # @!attribute should_parse + # When true, PDF pages are fetched and parsed. When false, PDF pages are skipped. + # + # @return [Boolean, nil] + optional :should_parse, ContextDev::Internal::Type::Boolean, api_name: :shouldParse + + # @!attribute start + # First 1-based PDF page to parse. + # + # @return [Integer, nil] + optional :start, Integer + + # @!method initialize(end_: nil, should_parse: nil, start: nil) + # Some parameter documentations has been truncated, see + # {ContextDev::Models::WebExtractParams::Pdf} for more details. + # + # @param end_ [Integer] Last 1-based PDF page to parse. Must be greater than or equal to start when both + # + # @param should_parse [Boolean] When true, PDF pages are fetched and parsed. When false, PDF pages are skipped. + # + # @param start [Integer] First 1-based PDF page to parse. + end + end + end +end diff --git a/lib/context_dev/models/web_extract_response.rb b/lib/context_dev/models/web_extract_response.rb new file mode 100644 index 00000000..93e1dfe5 --- /dev/null +++ b/lib/context_dev/models/web_extract_response.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module ContextDev + module Models + # @see ContextDev::Resources::Web#extract + class WebExtractResponse < ContextDev::Internal::Type::BaseModel + # @!attribute data + # Extracted data matching the request schema + # + # @return [Hash{Symbol=>Object}] + required :data, ContextDev::Internal::Type::HashOf[ContextDev::Internal::Type::Unknown] + + # @!attribute metadata + # + # @return [ContextDev::Models::WebExtractResponse::Metadata] + required :metadata, -> { ContextDev::Models::WebExtractResponse::Metadata } + + # @!attribute status + # Status of the response, e.g., 'ok' + # + # @return [String] + required :status, String + + # @!attribute url + # The starting URL that was analyzed + # + # @return [String] + required :url, String + + # @!attribute urls_analyzed + # List of URLs whose Markdown was used for extraction + # + # @return [Array] + required :urls_analyzed, ContextDev::Internal::Type::ArrayOf[String] + + # @!method initialize(data:, metadata:, status:, url:, urls_analyzed:) + # @param data [Hash{Symbol=>Object}] Extracted data matching the request schema + # + # @param metadata [ContextDev::Models::WebExtractResponse::Metadata] + # + # @param status [String] Status of the response, e.g., 'ok' + # + # @param url [String] The starting URL that was analyzed + # + # @param urls_analyzed [Array] List of URLs whose Markdown was used for extraction + + # @see ContextDev::Models::WebExtractResponse#metadata + class Metadata < ContextDev::Internal::Type::BaseModel + # @!attribute max_crawl_depth + # + # @return [Integer] + required :max_crawl_depth, Integer, api_name: :maxCrawlDepth + + # @!attribute num_failed + # + # @return [Integer] + required :num_failed, Integer, api_name: :numFailed + + # @!attribute num_skipped + # + # @return [Integer] + required :num_skipped, Integer, api_name: :numSkipped + + # @!attribute num_succeeded + # + # @return [Integer] + required :num_succeeded, Integer, api_name: :numSucceeded + + # @!attribute num_urls + # + # @return [Integer] + required :num_urls, Integer, api_name: :numUrls + + # @!method initialize(max_crawl_depth:, num_failed:, num_skipped:, num_succeeded:, num_urls:) + # @param max_crawl_depth [Integer] + # @param num_failed [Integer] + # @param num_skipped [Integer] + # @param num_succeeded [Integer] + # @param num_urls [Integer] + end + end + end +end diff --git a/lib/context_dev/resources/web.rb b/lib/context_dev/resources/web.rb index 351f4349..dba716d5 100644 --- a/lib/context_dev/resources/web.rb +++ b/lib/context_dev/resources/web.rb @@ -3,6 +3,54 @@ module ContextDev module Resources class Web + # Some parameter documentations has been truncated, see + # {ContextDev::Models::WebExtractParams} for more details. + # + # Crawl a website, convert pages to Markdown using the scrape cache, and extract + # structured data into the provided JSON Schema. The schema must describe the + # response data object. This endpoint does not accept targeted page-type + # selection. + # + # @overload extract(schema:, url:, fact_check: nil, follow_subdomains: nil, include_frames: nil, instructions: nil, max_age_ms: nil, pdf: nil, stop_after_ms: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {}) + # + # @param schema [Hash{Symbol=>Object}] JSON Schema for the returned data object. TypeScript Zod users can pass a JSON S + # + # @param url [String] The starting website URL to crawl and extract from. Must include http:// or http + # + # @param fact_check [Boolean] When true (default), every returned value must be grounded in facts stated on th + # + # @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain. + # + # @param include_frames [Boolean] When true, iframe contents are included in Markdown before extraction. + # + # @param instructions [String] Optional extraction guidance, such as which facts to prioritize or how to interp + # + # @param max_age_ms [Integer] Return cached scrape results if a prior scrape for the same parameters is younge + # + # @param pdf [ContextDev::Models::WebExtractParams::Pdf] + # + # @param stop_after_ms [Integer] Soft time budget for the crawl in milliseconds. + # + # @param timeout_ms [Integer] Optional timeout in milliseconds for the request. If the request takes longer th + # + # @param wait_for_ms [Integer] Optional browser wait time in milliseconds after initial page load for each craw + # + # @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}, nil] + # + # @return [ContextDev::Models::WebExtractResponse] + # + # @see ContextDev::Models::WebExtractParams + def extract(params) + parsed, options = ContextDev::WebExtractParams.dump_request(params) + @client.request( + method: :post, + path: "web/extract", + body: parsed, + model: ContextDev::Models::WebExtractResponse, + options: options + ) + end + # Some parameter documentations has been truncated, see # {ContextDev::Models::WebExtractFontsParams} for more details. # diff --git a/lib/context_dev/version.rb b/lib/context_dev/version.rb index 1947f293..9a2b056b 100644 --- a/lib/context_dev/version.rb +++ b/lib/context_dev/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ContextDev - VERSION = "1.24.0" + VERSION = "1.25.0" end diff --git a/rbi/context_dev/models.rbi b/rbi/context_dev/models.rbi index 1d62ec2a..147b94fa 100644 --- a/rbi/context_dev/models.rbi +++ b/rbi/context_dev/models.rbi @@ -34,6 +34,8 @@ module ContextDev WebExtractFontsParams = ContextDev::Models::WebExtractFontsParams + WebExtractParams = ContextDev::Models::WebExtractParams + WebExtractStyleguideParams = ContextDev::Models::WebExtractStyleguideParams WebScreenshotParams = ContextDev::Models::WebScreenshotParams diff --git a/rbi/context_dev/models/web_extract_params.rbi b/rbi/context_dev/models/web_extract_params.rbi new file mode 100644 index 00000000..97e242b1 --- /dev/null +++ b/rbi/context_dev/models/web_extract_params.rbi @@ -0,0 +1,232 @@ +# typed: strong + +module ContextDev + module Models + class WebExtractParams < ContextDev::Internal::Type::BaseModel + extend ContextDev::Internal::Type::RequestParameters::Converter + include ContextDev::Internal::Type::RequestParameters + + OrHash = + T.type_alias do + T.any(ContextDev::WebExtractParams, ContextDev::Internal::AnyHash) + end + + # JSON Schema for the returned data object. TypeScript Zod users can pass a JSON + # Schema generated from a Zod object; Python users can pass the equivalent JSON + # Schema object. + sig { returns(T::Hash[Symbol, T.anything]) } + attr_accessor :schema + + # The starting website URL to crawl and extract from. Must include http:// or + # https://. + sig { returns(String) } + attr_accessor :url + + # When true (default), every returned value must be grounded in facts stated on + # the page; fields that cannot be supported by the page are returned as + # null/empty. When false, the model may make reasonable inferences and derivations + # from the page content (e.g. ideal customer, competitor analysis, + # recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, + # metrics) faithful to the source. + sig { returns(T.nilable(T::Boolean)) } + attr_reader :fact_check + + sig { params(fact_check: T::Boolean).void } + attr_writer :fact_check + + # When true, follow links on subdomains of the starting URL's domain. + sig { returns(T.nilable(T::Boolean)) } + attr_reader :follow_subdomains + + sig { params(follow_subdomains: T::Boolean).void } + attr_writer :follow_subdomains + + # When true, iframe contents are included in Markdown before extraction. + sig { returns(T.nilable(T::Boolean)) } + attr_reader :include_frames + + sig { params(include_frames: T::Boolean).void } + attr_writer :include_frames + + # Optional extraction guidance, such as which facts to prioritize or how to + # interpret fields in the schema. + sig { returns(T.nilable(String)) } + attr_reader :instructions + + sig { params(instructions: String).void } + attr_writer :instructions + + # Return cached scrape results if a prior scrape for the same parameters is + # younger than this many milliseconds. + sig { returns(T.nilable(Integer)) } + attr_reader :max_age_ms + + sig { params(max_age_ms: Integer).void } + attr_writer :max_age_ms + + sig { returns(T.nilable(ContextDev::WebExtractParams::Pdf)) } + attr_reader :pdf + + sig { params(pdf: ContextDev::WebExtractParams::Pdf::OrHash).void } + attr_writer :pdf + + # Soft time budget for the crawl in milliseconds. + sig { returns(T.nilable(Integer)) } + attr_reader :stop_after_ms + + sig { params(stop_after_ms: Integer).void } + attr_writer :stop_after_ms + + # Optional timeout in milliseconds for the request. If the request takes longer + # than this value, it will be aborted with a 408 status code. Maximum allowed + # value is 300000ms (5 minutes). + sig { returns(T.nilable(Integer)) } + attr_reader :timeout_ms + + sig { params(timeout_ms: Integer).void } + attr_writer :timeout_ms + + # Optional browser wait time in milliseconds after initial page load for each + # crawled page. + sig { returns(T.nilable(Integer)) } + attr_reader :wait_for_ms + + sig { params(wait_for_ms: Integer).void } + attr_writer :wait_for_ms + + sig do + params( + schema: T::Hash[Symbol, T.anything], + url: String, + fact_check: T::Boolean, + follow_subdomains: T::Boolean, + include_frames: T::Boolean, + instructions: String, + max_age_ms: Integer, + pdf: ContextDev::WebExtractParams::Pdf::OrHash, + stop_after_ms: Integer, + timeout_ms: Integer, + wait_for_ms: Integer, + request_options: ContextDev::RequestOptions::OrHash + ).returns(T.attached_class) + end + def self.new( + # JSON Schema for the returned data object. TypeScript Zod users can pass a JSON + # Schema generated from a Zod object; Python users can pass the equivalent JSON + # Schema object. + schema:, + # The starting website URL to crawl and extract from. Must include http:// or + # https://. + url:, + # When true (default), every returned value must be grounded in facts stated on + # the page; fields that cannot be supported by the page are returned as + # null/empty. When false, the model may make reasonable inferences and derivations + # from the page content (e.g. ideal customer, competitor analysis, + # recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, + # metrics) faithful to the source. + fact_check: nil, + # When true, follow links on subdomains of the starting URL's domain. + follow_subdomains: nil, + # When true, iframe contents are included in Markdown before extraction. + include_frames: nil, + # Optional extraction guidance, such as which facts to prioritize or how to + # interpret fields in the schema. + instructions: nil, + # Return cached scrape results if a prior scrape for the same parameters is + # younger than this many milliseconds. + max_age_ms: nil, + pdf: nil, + # Soft time budget for the crawl in milliseconds. + stop_after_ms: nil, + # Optional timeout in milliseconds for the request. If the request takes longer + # than this value, it will be aborted with a 408 status code. Maximum allowed + # value is 300000ms (5 minutes). + timeout_ms: nil, + # Optional browser wait time in milliseconds after initial page load for each + # crawled page. + wait_for_ms: nil, + request_options: {} + ) + end + + sig do + override.returns( + { + schema: T::Hash[Symbol, T.anything], + url: String, + fact_check: T::Boolean, + follow_subdomains: T::Boolean, + include_frames: T::Boolean, + instructions: String, + max_age_ms: Integer, + pdf: ContextDev::WebExtractParams::Pdf, + stop_after_ms: Integer, + timeout_ms: Integer, + wait_for_ms: Integer, + request_options: ContextDev::RequestOptions + } + ) + end + def to_hash + end + + class Pdf < ContextDev::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any( + ContextDev::WebExtractParams::Pdf, + ContextDev::Internal::AnyHash + ) + end + + # Last 1-based PDF page to parse. Must be greater than or equal to start when both + # are provided. + sig { returns(T.nilable(Integer)) } + attr_reader :end_ + + sig { params(end_: Integer).void } + attr_writer :end_ + + # When true, PDF pages are fetched and parsed. When false, PDF pages are skipped. + sig { returns(T.nilable(T::Boolean)) } + attr_reader :should_parse + + sig { params(should_parse: T::Boolean).void } + attr_writer :should_parse + + # First 1-based PDF page to parse. + sig { returns(T.nilable(Integer)) } + attr_reader :start + + sig { params(start: Integer).void } + attr_writer :start + + sig do + params( + end_: Integer, + should_parse: T::Boolean, + start: Integer + ).returns(T.attached_class) + end + def self.new( + # Last 1-based PDF page to parse. Must be greater than or equal to start when both + # are provided. + end_: nil, + # When true, PDF pages are fetched and parsed. When false, PDF pages are skipped. + should_parse: nil, + # First 1-based PDF page to parse. + start: nil + ) + end + + sig do + override.returns( + { end_: Integer, should_parse: T::Boolean, start: Integer } + ) + end + def to_hash + end + end + end + end +end diff --git a/rbi/context_dev/models/web_extract_response.rbi b/rbi/context_dev/models/web_extract_response.rbi new file mode 100644 index 00000000..4b81ac9a --- /dev/null +++ b/rbi/context_dev/models/web_extract_response.rbi @@ -0,0 +1,134 @@ +# typed: strong + +module ContextDev + module Models + class WebExtractResponse < ContextDev::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any( + ContextDev::Models::WebExtractResponse, + ContextDev::Internal::AnyHash + ) + end + + # Extracted data matching the request schema + sig { returns(T::Hash[Symbol, T.anything]) } + attr_accessor :data + + sig { returns(ContextDev::Models::WebExtractResponse::Metadata) } + attr_reader :metadata + + sig do + params( + metadata: ContextDev::Models::WebExtractResponse::Metadata::OrHash + ).void + end + attr_writer :metadata + + # Status of the response, e.g., 'ok' + sig { returns(String) } + attr_accessor :status + + # The starting URL that was analyzed + sig { returns(String) } + attr_accessor :url + + # List of URLs whose Markdown was used for extraction + sig { returns(T::Array[String]) } + attr_accessor :urls_analyzed + + sig do + params( + data: T::Hash[Symbol, T.anything], + metadata: ContextDev::Models::WebExtractResponse::Metadata::OrHash, + status: String, + url: String, + urls_analyzed: T::Array[String] + ).returns(T.attached_class) + end + def self.new( + # Extracted data matching the request schema + data:, + metadata:, + # Status of the response, e.g., 'ok' + status:, + # The starting URL that was analyzed + url:, + # List of URLs whose Markdown was used for extraction + urls_analyzed: + ) + end + + sig do + override.returns( + { + data: T::Hash[Symbol, T.anything], + metadata: ContextDev::Models::WebExtractResponse::Metadata, + status: String, + url: String, + urls_analyzed: T::Array[String] + } + ) + end + def to_hash + end + + class Metadata < ContextDev::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any( + ContextDev::Models::WebExtractResponse::Metadata, + ContextDev::Internal::AnyHash + ) + end + + sig { returns(Integer) } + attr_accessor :max_crawl_depth + + sig { returns(Integer) } + attr_accessor :num_failed + + sig { returns(Integer) } + attr_accessor :num_skipped + + sig { returns(Integer) } + attr_accessor :num_succeeded + + sig { returns(Integer) } + attr_accessor :num_urls + + sig do + params( + max_crawl_depth: Integer, + num_failed: Integer, + num_skipped: Integer, + num_succeeded: Integer, + num_urls: Integer + ).returns(T.attached_class) + end + def self.new( + max_crawl_depth:, + num_failed:, + num_skipped:, + num_succeeded:, + num_urls: + ) + end + + sig do + override.returns( + { + max_crawl_depth: Integer, + num_failed: Integer, + num_skipped: Integer, + num_succeeded: Integer, + num_urls: Integer + } + ) + end + def to_hash + end + end + end + end +end diff --git a/rbi/context_dev/resources/web.rbi b/rbi/context_dev/resources/web.rbi index c56745b9..92d7ba1e 100644 --- a/rbi/context_dev/resources/web.rbi +++ b/rbi/context_dev/resources/web.rbi @@ -3,6 +3,65 @@ module ContextDev module Resources class Web + # Crawl a website, convert pages to Markdown using the scrape cache, and extract + # structured data into the provided JSON Schema. The schema must describe the + # response data object. This endpoint does not accept targeted page-type + # selection. + sig do + params( + schema: T::Hash[Symbol, T.anything], + url: String, + fact_check: T::Boolean, + follow_subdomains: T::Boolean, + include_frames: T::Boolean, + instructions: String, + max_age_ms: Integer, + pdf: ContextDev::WebExtractParams::Pdf::OrHash, + stop_after_ms: Integer, + timeout_ms: Integer, + wait_for_ms: Integer, + request_options: ContextDev::RequestOptions::OrHash + ).returns(ContextDev::Models::WebExtractResponse) + end + def extract( + # JSON Schema for the returned data object. TypeScript Zod users can pass a JSON + # Schema generated from a Zod object; Python users can pass the equivalent JSON + # Schema object. + schema:, + # The starting website URL to crawl and extract from. Must include http:// or + # https://. + url:, + # When true (default), every returned value must be grounded in facts stated on + # the page; fields that cannot be supported by the page are returned as + # null/empty. When false, the model may make reasonable inferences and derivations + # from the page content (e.g. ideal customer, competitor analysis, + # recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, + # metrics) faithful to the source. + fact_check: nil, + # When true, follow links on subdomains of the starting URL's domain. + follow_subdomains: nil, + # When true, iframe contents are included in Markdown before extraction. + include_frames: nil, + # Optional extraction guidance, such as which facts to prioritize or how to + # interpret fields in the schema. + instructions: nil, + # Return cached scrape results if a prior scrape for the same parameters is + # younger than this many milliseconds. + max_age_ms: nil, + pdf: nil, + # Soft time budget for the crawl in milliseconds. + stop_after_ms: nil, + # Optional timeout in milliseconds for the request. If the request takes longer + # than this value, it will be aborted with a 408 status code. Maximum allowed + # value is 300000ms (5 minutes). + timeout_ms: nil, + # Optional browser wait time in milliseconds after initial page load for each + # crawled page. + wait_for_ms: nil, + request_options: {} + ) + end + # Scrape font information from a website including font families, usage # statistics, fallbacks, and element/word counts. sig do diff --git a/sig/context_dev/models.rbs b/sig/context_dev/models.rbs index 2292386d..322e9ff9 100644 --- a/sig/context_dev/models.rbs +++ b/sig/context_dev/models.rbs @@ -29,6 +29,8 @@ module ContextDev class WebExtractFontsParams = ContextDev::Models::WebExtractFontsParams + class WebExtractParams = ContextDev::Models::WebExtractParams + class WebExtractStyleguideParams = ContextDev::Models::WebExtractStyleguideParams class WebScreenshotParams = ContextDev::Models::WebScreenshotParams diff --git a/sig/context_dev/models/web_extract_params.rbs b/sig/context_dev/models/web_extract_params.rbs new file mode 100644 index 00000000..e8561d97 --- /dev/null +++ b/sig/context_dev/models/web_extract_params.rbs @@ -0,0 +1,120 @@ +module ContextDev + module Models + type web_extract_params = + { + schema: ::Hash[Symbol, top], + url: String, + fact_check: bool, + follow_subdomains: bool, + include_frames: bool, + instructions: String, + max_age_ms: Integer, + pdf: ContextDev::WebExtractParams::Pdf, + stop_after_ms: Integer, + timeout_ms: Integer, + wait_for_ms: Integer + } + & ContextDev::Internal::Type::request_parameters + + class WebExtractParams < ContextDev::Internal::Type::BaseModel + extend ContextDev::Internal::Type::RequestParameters::Converter + include ContextDev::Internal::Type::RequestParameters + + attr_accessor schema: ::Hash[Symbol, top] + + attr_accessor url: String + + attr_reader fact_check: bool? + + def fact_check=: (bool) -> bool + + attr_reader follow_subdomains: bool? + + def follow_subdomains=: (bool) -> bool + + attr_reader include_frames: bool? + + def include_frames=: (bool) -> bool + + attr_reader instructions: String? + + def instructions=: (String) -> String + + attr_reader max_age_ms: Integer? + + def max_age_ms=: (Integer) -> Integer + + attr_reader pdf: ContextDev::WebExtractParams::Pdf? + + def pdf=: ( + ContextDev::WebExtractParams::Pdf + ) -> ContextDev::WebExtractParams::Pdf + + attr_reader stop_after_ms: Integer? + + def stop_after_ms=: (Integer) -> Integer + + attr_reader timeout_ms: Integer? + + def timeout_ms=: (Integer) -> Integer + + attr_reader wait_for_ms: Integer? + + def wait_for_ms=: (Integer) -> Integer + + def initialize: ( + schema: ::Hash[Symbol, top], + url: String, + ?fact_check: bool, + ?follow_subdomains: bool, + ?include_frames: bool, + ?instructions: String, + ?max_age_ms: Integer, + ?pdf: ContextDev::WebExtractParams::Pdf, + ?stop_after_ms: Integer, + ?timeout_ms: Integer, + ?wait_for_ms: Integer, + ?request_options: ContextDev::request_opts + ) -> void + + def to_hash: -> { + schema: ::Hash[Symbol, top], + url: String, + fact_check: bool, + follow_subdomains: bool, + include_frames: bool, + instructions: String, + max_age_ms: Integer, + pdf: ContextDev::WebExtractParams::Pdf, + stop_after_ms: Integer, + timeout_ms: Integer, + wait_for_ms: Integer, + request_options: ContextDev::RequestOptions + } + + type pdf = { end_: Integer, should_parse: bool, start: Integer } + + class Pdf < ContextDev::Internal::Type::BaseModel + attr_reader end_: Integer? + + def end_=: (Integer) -> Integer + + attr_reader should_parse: bool? + + def should_parse=: (bool) -> bool + + attr_reader start: Integer? + + def start=: (Integer) -> Integer + + def initialize: ( + ?end_: Integer, + ?should_parse: bool, + ?start: Integer + ) -> void + + def to_hash: -> { end_: Integer, should_parse: bool, start: Integer } + end + end + end +end diff --git a/sig/context_dev/models/web_extract_response.rbs b/sig/context_dev/models/web_extract_response.rbs new file mode 100644 index 00000000..34798d30 --- /dev/null +++ b/sig/context_dev/models/web_extract_response.rbs @@ -0,0 +1,77 @@ +module ContextDev + module Models + type web_extract_response = + { + data: ::Hash[Symbol, top], + metadata: ContextDev::Models::WebExtractResponse::Metadata, + status: String, + url: String, + urls_analyzed: ::Array[String] + } + + class WebExtractResponse < ContextDev::Internal::Type::BaseModel + attr_accessor data: ::Hash[Symbol, top] + + attr_accessor metadata: ContextDev::Models::WebExtractResponse::Metadata + + attr_accessor status: String + + attr_accessor url: String + + attr_accessor urls_analyzed: ::Array[String] + + def initialize: ( + data: ::Hash[Symbol, top], + metadata: ContextDev::Models::WebExtractResponse::Metadata, + status: String, + url: String, + urls_analyzed: ::Array[String] + ) -> void + + def to_hash: -> { + data: ::Hash[Symbol, top], + metadata: ContextDev::Models::WebExtractResponse::Metadata, + status: String, + url: String, + urls_analyzed: ::Array[String] + } + + type metadata = + { + max_crawl_depth: Integer, + num_failed: Integer, + num_skipped: Integer, + num_succeeded: Integer, + num_urls: Integer + } + + class Metadata < ContextDev::Internal::Type::BaseModel + attr_accessor max_crawl_depth: Integer + + attr_accessor num_failed: Integer + + attr_accessor num_skipped: Integer + + attr_accessor num_succeeded: Integer + + attr_accessor num_urls: Integer + + def initialize: ( + max_crawl_depth: Integer, + num_failed: Integer, + num_skipped: Integer, + num_succeeded: Integer, + num_urls: Integer + ) -> void + + def to_hash: -> { + max_crawl_depth: Integer, + num_failed: Integer, + num_skipped: Integer, + num_succeeded: Integer, + num_urls: Integer + } + end + end + end +end diff --git a/sig/context_dev/resources/web.rbs b/sig/context_dev/resources/web.rbs index 5eeac331..e2259474 100644 --- a/sig/context_dev/resources/web.rbs +++ b/sig/context_dev/resources/web.rbs @@ -1,6 +1,21 @@ module ContextDev module Resources class Web + def extract: ( + schema: ::Hash[Symbol, top], + url: String, + ?fact_check: bool, + ?follow_subdomains: bool, + ?include_frames: bool, + ?instructions: String, + ?max_age_ms: Integer, + ?pdf: ContextDev::WebExtractParams::Pdf, + ?stop_after_ms: Integer, + ?timeout_ms: Integer, + ?wait_for_ms: Integer, + ?request_options: ContextDev::request_opts + ) -> ContextDev::Models::WebExtractResponse + def extract_fonts: ( ?direct_url: String, ?domain: String, diff --git a/test/context_dev/resources/web_test.rb b/test/context_dev/resources/web_test.rb index 103b11fd..a1a5fb59 100644 --- a/test/context_dev/resources/web_test.rb +++ b/test/context_dev/resources/web_test.rb @@ -3,6 +3,30 @@ require_relative "../test_helper" class ContextDev::Test::Resources::WebTest < ContextDev::Test::ResourceTest + def test_extract_required_params + skip("Mock server tests are disabled") + + response = + @context_dev.web.extract( + schema: {type: "bar", properties: "bar", required: "bar", additionalProperties: "bar"}, + url: "https://example.com" + ) + + assert_pattern do + response => ContextDev::Models::WebExtractResponse + end + + assert_pattern do + response => { + data: ^(ContextDev::Internal::Type::HashOf[ContextDev::Internal::Type::Unknown]), + metadata: ContextDev::Models::WebExtractResponse::Metadata, + status: String, + url: String, + urls_analyzed: ^(ContextDev::Internal::Type::ArrayOf[String]) + } + end + end + def test_extract_fonts skip("Mock server tests are disabled")