From c897bbf03235960c8036bd808c5239b935d94bf3 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Sun, 7 Jun 2026 22:03:09 +0000 Subject: [PATCH 1/2] feat(api): api update --- .stats.yml | 4 +-- .../models/web_web_crawl_md_params.rb | 23 ++++++++++++- .../models/web_web_scrape_html_params.rb | 22 ++++++++++++- .../models/web_web_scrape_md_params.rb | 22 ++++++++++++- lib/context_dev/resources/web.rb | 22 +++++++++++-- .../models/web_web_crawl_md_params.rbi | 32 +++++++++++++++++++ .../models/web_web_scrape_html_params.rbi | 30 +++++++++++++++++ .../models/web_web_scrape_md_params.rbi | 30 +++++++++++++++++ rbi/context_dev/resources/web.rbi | 31 ++++++++++++++++++ .../models/web_web_crawl_md_params.rbs | 14 ++++++++ .../models/web_web_scrape_html_params.rbs | 14 ++++++++ .../models/web_web_scrape_md_params.rbs | 14 ++++++++ sig/context_dev/resources/web.rbs | 6 ++++ 13 files changed, 256 insertions(+), 8 deletions(-) diff --git a/.stats.yml b/.stats.yml index 27382d2f..f1634d6e 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 25 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-de0ba2ed18ec7265b0b20d9639b3da207dd33e354f0a856091902291c6661830.yml -openapi_spec_hash: ae25efc2ac6bcab54bc3f4efaab73a55 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-8f6bc62256b5a4aa44e82389bfefee1d73172856c407567bba11a82fd4203641.yml +openapi_spec_hash: 8b7230de56ecbec4712fa1e81067a75c config_hash: f86a4e06ae5ed725aa79d58d7a8dc27c diff --git a/lib/context_dev/models/web_web_crawl_md_params.rb b/lib/context_dev/models/web_web_crawl_md_params.rb index 28560505..b8f504ea 100644 --- a/lib/context_dev/models/web_web_crawl_md_params.rb +++ b/lib/context_dev/models/web_web_crawl_md_params.rb @@ -13,6 +13,14 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel # @return [String] required :url, String + # @!attribute exclude_selectors + # CSS selectors to remove before each crawled page is converted to Markdown. + # Applied after includeSelectors. Exclusion takes precedence: an element matching + # both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + # + # @return [Array, nil] + optional :exclude_selectors, ContextDev::Internal::Type::ArrayOf[String], api_name: :excludeSelectors + # @!attribute follow_subdomains # When true, follow links on subdomains of the starting URL's domain (e.g. # docs.example.com when starting from example.com). www and apex are always @@ -40,6 +48,15 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel # @return [Boolean, nil] optional :include_links, ContextDev::Internal::Type::Boolean, api_name: :includeLinks + # @!attribute include_selectors + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before each crawled page is converted to Markdown. When + # omitted, the entire document is kept. Examples: "article.main", "#content", + # "[role=main]". + # + # @return [Array, nil] + optional :include_selectors, ContextDev::Internal::Type::ArrayOf[String], api_name: :includeSelectors + # @!attribute max_age_ms # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when @@ -110,12 +127,14 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel # @return [Integer, nil] optional :wait_for_ms, Integer, api_name: :waitForMs - # @!method initialize(url:, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, shorten_base64_images: nil, stop_after_ms: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) + # @!method initialize(url:, exclude_selectors: nil, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, include_selectors: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, shorten_base64_images: nil, stop_after_ms: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) # Some parameter documentations has been truncated, see # {ContextDev::Models::WebWebCrawlMdParams} for more details. # # @param url [String] The starting URL for the crawl (must include http:// or https:// protocol) # + # @param exclude_selectors [Array] CSS selectors to remove before each crawled page is converted to Markdown. Appli + # # @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex # # @param include_frames [Boolean] When true, the contents of iframes are rendered to Markdown for each crawled pag @@ -124,6 +143,8 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel # # @param include_links [Boolean] Preserve hyperlinks in the Markdown output # + # @param include_selectors [Array] CSS selectors. When provided, only matching HTML subtrees (and their descendants + # # @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y # # @param max_depth [Integer] Maximum link depth from the starting URL (0 = only the starting page) diff --git a/lib/context_dev/models/web_web_scrape_html_params.rb b/lib/context_dev/models/web_web_scrape_html_params.rb index 781f40be..fc136a86 100644 --- a/lib/context_dev/models/web_web_scrape_html_params.rb +++ b/lib/context_dev/models/web_web_scrape_html_params.rb @@ -13,6 +13,14 @@ class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel # @return [String] required :url, String + # @!attribute exclude_selectors + # CSS selectors to remove from the result. Applied after includeSelectors. + # Exclusion takes precedence: an element matching both is removed. Examples: + # "nav", "footer", ".ad-banner", "[aria-hidden=true]". + # + # @return [Array, nil] + optional :exclude_selectors, ContextDev::Internal::Type::ArrayOf[String] + # @!attribute headers # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching @@ -27,6 +35,14 @@ class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel # @return [Boolean, nil] optional :include_frames, ContextDev::Internal::Type::Boolean + # @!attribute include_selectors + # CSS selectors. When provided, only matching subtrees (and their descendants) are + # kept and everything else is dropped. When omitted, the entire document is kept. + # Examples: "article.main", "#content", "[role=main]". + # + # @return [Array, nil] + optional :include_selectors, ContextDev::Internal::Type::ArrayOf[String] + # @!attribute max_age_ms # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when @@ -57,16 +73,20 @@ class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel # @return [Integer, nil] optional :wait_for_ms, Integer - # @!method initialize(url:, headers: nil, include_frames: nil, max_age_ms: nil, pdf: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {}) + # @!method initialize(url:, exclude_selectors: nil, headers: nil, include_frames: nil, include_selectors: nil, max_age_ms: nil, pdf: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {}) # Some parameter documentations has been truncated, see # {ContextDev::Models::WebWebScrapeHTMLParams} for more details. # # @param url [String] Full URL to scrape (must include http:// or https:// protocol) # + # @param exclude_selectors [Array] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi + # # @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob # # @param include_frames [Boolean] When true, iframes are rendered inline into the returned HTML. # + # @param include_selectors [Array] CSS selectors. When provided, only matching subtrees (and their descendants) are + # # @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y # # @param pdf [ContextDev::Models::WebWebScrapeHTMLParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu diff --git a/lib/context_dev/models/web_web_scrape_md_params.rb b/lib/context_dev/models/web_web_scrape_md_params.rb index b899574c..40fe595c 100644 --- a/lib/context_dev/models/web_web_scrape_md_params.rb +++ b/lib/context_dev/models/web_web_scrape_md_params.rb @@ -14,6 +14,14 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel # @return [String] required :url, String + # @!attribute exclude_selectors + # CSS selectors to remove before conversion to Markdown. Applied after + # includeSelectors. Exclusion takes precedence: an element matching both is + # removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + # + # @return [Array, nil] + optional :exclude_selectors, ContextDev::Internal::Type::ArrayOf[String] + # @!attribute headers # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching @@ -40,6 +48,14 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel # @return [Boolean, nil] optional :include_links, ContextDev::Internal::Type::Boolean + # @!attribute include_selectors + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before conversion to Markdown. When omitted, the entire + # document is kept. Examples: "article.main", "#content", "[role=main]". + # + # @return [Array, nil] + optional :include_selectors, ContextDev::Internal::Type::ArrayOf[String] + # @!attribute max_age_ms # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when @@ -83,12 +99,14 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel # @return [Integer, nil] optional :wait_for_ms, Integer - # @!method initialize(url:, headers: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, pdf: nil, shorten_base64_images: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) + # @!method initialize(url:, exclude_selectors: nil, headers: nil, include_frames: nil, include_images: nil, include_links: nil, include_selectors: nil, max_age_ms: nil, pdf: nil, shorten_base64_images: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) # Some parameter documentations has been truncated, see # {ContextDev::Models::WebWebScrapeMdParams} for more details. # # @param url [String] Full URL to scrape into LLM usable Markdown (must include http:// or https:// pr # + # @param exclude_selectors [Array] CSS selectors to remove before conversion to Markdown. Applied after includeSele + # # @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob # # @param include_frames [Boolean] When true, the contents of iframes are rendered to Markdown. @@ -97,6 +115,8 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel # # @param include_links [Boolean] Preserve hyperlinks in Markdown output # + # @param include_selectors [Array] CSS selectors. When provided, only matching HTML subtrees (and their descendants + # # @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y # # @param pdf [ContextDev::Models::WebWebScrapeMdParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu diff --git a/lib/context_dev/resources/web.rb b/lib/context_dev/resources/web.rb index 1fb1dc3e..7e6d12aa 100644 --- a/lib/context_dev/resources/web.rb +++ b/lib/context_dev/resources/web.rb @@ -246,10 +246,12 @@ def search(params) # Performs a crawl starting from a given URL, extracts page content as Markdown, # and returns results for all crawled pages. # - # @overload web_crawl_md(url:, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, shorten_base64_images: nil, stop_after_ms: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) + # @overload web_crawl_md(url:, exclude_selectors: nil, follow_subdomains: nil, include_frames: nil, include_images: nil, include_links: nil, include_selectors: nil, max_age_ms: nil, max_depth: nil, max_pages: nil, pdf: nil, shorten_base64_images: nil, stop_after_ms: nil, timeout_ms: nil, url_regex: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) # # @param url [String] The starting URL for the crawl (must include http:// or https:// protocol) # + # @param exclude_selectors [Array] CSS selectors to remove before each crawled page is converted to Markdown. Appli + # # @param follow_subdomains [Boolean] When true, follow links on subdomains of the starting URL's domain (e.g. docs.ex # # @param include_frames [Boolean] When true, the contents of iframes are rendered to Markdown for each crawled pag @@ -258,6 +260,8 @@ def search(params) # # @param include_links [Boolean] Preserve hyperlinks in the Markdown output # + # @param include_selectors [Array] CSS selectors. When provided, only matching HTML subtrees (and their descendants + # # @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y # # @param max_depth [Integer] Maximum link depth from the starting URL (0 = only the starting page) @@ -299,14 +303,18 @@ def web_crawl_md(params) # # Scrapes the given URL and returns the raw HTML content of the page. # - # @overload web_scrape_html(url:, headers: nil, include_frames: nil, max_age_ms: nil, pdf: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {}) + # @overload web_scrape_html(url:, exclude_selectors: nil, headers: nil, include_frames: nil, include_selectors: nil, max_age_ms: nil, pdf: nil, timeout_ms: nil, wait_for_ms: nil, request_options: {}) # # @param url [String] Full URL to scrape (must include http:// or https:// protocol) # + # @param exclude_selectors [Array] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi + # # @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob # # @param include_frames [Boolean] When true, iframes are rendered inline into the returned HTML. # + # @param include_selectors [Array] CSS selectors. When provided, only matching subtrees (and their descendants) are + # # @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y # # @param pdf [ContextDev::Models::WebWebScrapeHTMLParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu @@ -327,7 +335,9 @@ def web_scrape_html(params) method: :get, path: "web/scrape/html", query: query.transform_keys( + exclude_selectors: "excludeSelectors", include_frames: "includeFrames", + include_selectors: "includeSelectors", max_age_ms: "maxAgeMs", timeout_ms: "timeoutMS", wait_for_ms: "waitForMs" @@ -385,10 +395,12 @@ def web_scrape_images(params) # # Scrapes the given URL into LLM usable Markdown. # - # @overload web_scrape_md(url:, headers: nil, include_frames: nil, include_images: nil, include_links: nil, max_age_ms: nil, pdf: nil, shorten_base64_images: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) + # @overload web_scrape_md(url:, exclude_selectors: nil, headers: nil, include_frames: nil, include_images: nil, include_links: nil, include_selectors: nil, max_age_ms: nil, pdf: nil, shorten_base64_images: nil, timeout_ms: nil, use_main_content_only: nil, wait_for_ms: nil, request_options: {}) # # @param url [String] Full URL to scrape into LLM usable Markdown (must include http:// or https:// pr # + # @param exclude_selectors [Array] CSS selectors to remove before conversion to Markdown. Applied after includeSele + # # @param headers [Hash{Symbol=>String}] Optional outbound HTTP headers forwarded only to the target URL, sent as deep-ob # # @param include_frames [Boolean] When true, the contents of iframes are rendered to Markdown. @@ -397,6 +409,8 @@ def web_scrape_images(params) # # @param include_links [Boolean] Preserve hyperlinks in Markdown output # + # @param include_selectors [Array] CSS selectors. When provided, only matching HTML subtrees (and their descendants + # # @param max_age_ms [Integer] Return a cached result if a prior scrape for the same parameters exists and is y # # @param pdf [ContextDev::Models::WebWebScrapeMdParams::Pdf] PDF parsing controls. Use start/end to limit text extraction and OCR to an inclu @@ -421,9 +435,11 @@ def web_scrape_md(params) method: :get, path: "web/scrape/markdown", query: query.transform_keys( + exclude_selectors: "excludeSelectors", include_frames: "includeFrames", include_images: "includeImages", include_links: "includeLinks", + include_selectors: "includeSelectors", max_age_ms: "maxAgeMs", shorten_base64_images: "shortenBase64Images", timeout_ms: "timeoutMS", diff --git a/rbi/context_dev/models/web_web_crawl_md_params.rbi b/rbi/context_dev/models/web_web_crawl_md_params.rbi index c3dc457b..9c6f5247 100644 --- a/rbi/context_dev/models/web_web_crawl_md_params.rbi +++ b/rbi/context_dev/models/web_web_crawl_md_params.rbi @@ -15,6 +15,15 @@ module ContextDev sig { returns(String) } attr_accessor :url + # CSS selectors to remove before each crawled page is converted to Markdown. + # Applied after includeSelectors. Exclusion takes precedence: an element matching + # both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + sig { returns(T.nilable(T::Array[String])) } + attr_reader :exclude_selectors + + sig { params(exclude_selectors: T::Array[String]).void } + attr_writer :exclude_selectors + # When true, follow links on subdomains of the starting URL's domain (e.g. # docs.example.com when starting from example.com). www and apex are always # treated as equivalent. @@ -46,6 +55,16 @@ module ContextDev sig { params(include_links: T::Boolean).void } attr_writer :include_links + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before each crawled page is converted to Markdown. When + # omitted, the entire document is kept. Examples: "article.main", "#content", + # "[role=main]". + sig { returns(T.nilable(T::Array[String])) } + attr_reader :include_selectors + + sig { params(include_selectors: T::Array[String]).void } + attr_writer :include_selectors + # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -129,10 +148,12 @@ module ContextDev sig do params( url: String, + exclude_selectors: T::Array[String], follow_subdomains: T::Boolean, include_frames: T::Boolean, include_images: T::Boolean, include_links: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, max_depth: Integer, max_pages: Integer, @@ -149,6 +170,10 @@ module ContextDev def self.new( # The starting URL for the crawl (must include http:// or https:// protocol) url:, + # CSS selectors to remove before each crawled page is converted to Markdown. + # Applied after includeSelectors. Exclusion takes precedence: an element matching + # both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + exclude_selectors: nil, # When true, follow links on subdomains of the starting URL's domain (e.g. # docs.example.com when starting from example.com). www and apex are always # treated as equivalent. @@ -160,6 +185,11 @@ module ContextDev include_images: nil, # Preserve hyperlinks in the Markdown output include_links: nil, + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before each crawled page is converted to Markdown. When + # omitted, the entire document is kept. Examples: "article.main", "#content", + # "[role=main]". + include_selectors: nil, # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -198,10 +228,12 @@ module ContextDev override.returns( { url: String, + exclude_selectors: T::Array[String], follow_subdomains: T::Boolean, include_frames: T::Boolean, include_images: T::Boolean, include_links: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, max_depth: Integer, max_pages: Integer, diff --git a/rbi/context_dev/models/web_web_scrape_html_params.rbi b/rbi/context_dev/models/web_web_scrape_html_params.rbi index c6273e37..ca227bd4 100644 --- a/rbi/context_dev/models/web_web_scrape_html_params.rbi +++ b/rbi/context_dev/models/web_web_scrape_html_params.rbi @@ -18,6 +18,15 @@ module ContextDev sig { returns(String) } attr_accessor :url + # CSS selectors to remove from the result. Applied after includeSelectors. + # Exclusion takes precedence: an element matching both is removed. Examples: + # "nav", "footer", ".ad-banner", "[aria-hidden=true]". + sig { returns(T.nilable(T::Array[String])) } + attr_reader :exclude_selectors + + sig { params(exclude_selectors: T::Array[String]).void } + attr_writer :exclude_selectors + # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching # is bypassed: the result is neither read from nor written to cache. @@ -34,6 +43,15 @@ module ContextDev sig { params(include_frames: T::Boolean).void } attr_writer :include_frames + # CSS selectors. When provided, only matching subtrees (and their descendants) are + # kept and everything else is dropped. When omitted, the entire document is kept. + # Examples: "article.main", "#content", "[role=main]". + sig { returns(T.nilable(T::Array[String])) } + attr_reader :include_selectors + + sig { params(include_selectors: T::Array[String]).void } + attr_writer :include_selectors + # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -71,8 +89,10 @@ module ContextDev sig do params( url: String, + exclude_selectors: T::Array[String], headers: T::Hash[Symbol, String], include_frames: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeHTMLParams::Pdf::OrHash, timeout_ms: Integer, @@ -83,12 +103,20 @@ module ContextDev def self.new( # Full URL to scrape (must include http:// or https:// protocol) url:, + # CSS selectors to remove from the result. Applied after includeSelectors. + # Exclusion takes precedence: an element matching both is removed. Examples: + # "nav", "footer", ".ad-banner", "[aria-hidden=true]". + exclude_selectors: nil, # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching # is bypassed: the result is neither read from nor written to cache. headers: nil, # When true, iframes are rendered inline into the returned HTML. include_frames: nil, + # CSS selectors. When provided, only matching subtrees (and their descendants) are + # kept and everything else is dropped. When omitted, the entire document is kept. + # Examples: "article.main", "#content", "[role=main]". + include_selectors: nil, # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -111,8 +139,10 @@ module ContextDev override.returns( { url: String, + exclude_selectors: T::Array[String], headers: T::Hash[Symbol, String], include_frames: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeHTMLParams::Pdf, timeout_ms: Integer, diff --git a/rbi/context_dev/models/web_web_scrape_md_params.rbi b/rbi/context_dev/models/web_web_scrape_md_params.rbi index 558e0469..4e4d47dd 100644 --- a/rbi/context_dev/models/web_web_scrape_md_params.rbi +++ b/rbi/context_dev/models/web_web_scrape_md_params.rbi @@ -16,6 +16,15 @@ module ContextDev sig { returns(String) } attr_accessor :url + # CSS selectors to remove before conversion to Markdown. Applied after + # includeSelectors. Exclusion takes precedence: an element matching both is + # removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + sig { returns(T.nilable(T::Array[String])) } + attr_reader :exclude_selectors + + sig { params(exclude_selectors: T::Array[String]).void } + attr_writer :exclude_selectors + # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching # is bypassed: the result is neither read from nor written to cache. @@ -46,6 +55,15 @@ module ContextDev sig { params(include_links: T::Boolean).void } attr_writer :include_links + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before conversion to Markdown. When omitted, the entire + # document is kept. Examples: "article.main", "#content", "[role=main]". + sig { returns(T.nilable(T::Array[String])) } + attr_reader :include_selectors + + sig { params(include_selectors: T::Array[String]).void } + attr_writer :include_selectors + # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -98,10 +116,12 @@ module ContextDev sig do params( url: String, + exclude_selectors: T::Array[String], headers: T::Hash[Symbol, String], include_frames: T::Boolean, include_images: T::Boolean, include_links: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeMdParams::Pdf::OrHash, shorten_base64_images: T::Boolean, @@ -115,6 +135,10 @@ module ContextDev # Full URL to scrape into LLM usable Markdown (must include http:// or https:// # protocol) url:, + # CSS selectors to remove before conversion to Markdown. Applied after + # includeSelectors. Exclusion takes precedence: an element matching both is + # removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + exclude_selectors: nil, # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching # is bypassed: the result is neither read from nor written to cache. @@ -125,6 +149,10 @@ module ContextDev include_images: nil, # Preserve hyperlinks in Markdown output include_links: nil, + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before conversion to Markdown. When omitted, the entire + # document is kept. Examples: "article.main", "#content", "[role=main]". + include_selectors: nil, # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -152,10 +180,12 @@ module ContextDev override.returns( { url: String, + exclude_selectors: T::Array[String], headers: T::Hash[Symbol, String], include_frames: T::Boolean, include_images: T::Boolean, include_links: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeMdParams::Pdf, shorten_base64_images: T::Boolean, diff --git a/rbi/context_dev/resources/web.rbi b/rbi/context_dev/resources/web.rbi index 759c66a7..30f32706 100644 --- a/rbi/context_dev/resources/web.rbi +++ b/rbi/context_dev/resources/web.rbi @@ -251,10 +251,12 @@ module ContextDev sig do params( url: String, + exclude_selectors: T::Array[String], follow_subdomains: T::Boolean, include_frames: T::Boolean, include_images: T::Boolean, include_links: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, max_depth: Integer, max_pages: Integer, @@ -271,6 +273,10 @@ module ContextDev def web_crawl_md( # The starting URL for the crawl (must include http:// or https:// protocol) url:, + # CSS selectors to remove before each crawled page is converted to Markdown. + # Applied after includeSelectors. Exclusion takes precedence: an element matching + # both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + exclude_selectors: nil, # When true, follow links on subdomains of the starting URL's domain (e.g. # docs.example.com when starting from example.com). www and apex are always # treated as equivalent. @@ -282,6 +288,11 @@ module ContextDev include_images: nil, # Preserve hyperlinks in the Markdown output include_links: nil, + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before each crawled page is converted to Markdown. When + # omitted, the entire document is kept. Examples: "article.main", "#content", + # "[role=main]". + include_selectors: nil, # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -320,8 +331,10 @@ module ContextDev sig do params( url: String, + exclude_selectors: T::Array[String], headers: T::Hash[Symbol, String], include_frames: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeHTMLParams::Pdf::OrHash, timeout_ms: Integer, @@ -332,12 +345,20 @@ module ContextDev def web_scrape_html( # Full URL to scrape (must include http:// or https:// protocol) url:, + # CSS selectors to remove from the result. Applied after includeSelectors. + # Exclusion takes precedence: an element matching both is removed. Examples: + # "nav", "footer", ".ad-banner", "[aria-hidden=true]". + exclude_selectors: nil, # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching # is bypassed: the result is neither read from nor written to cache. headers: nil, # When true, iframes are rendered inline into the returned HTML. include_frames: nil, + # CSS selectors. When provided, only matching subtrees (and their descendants) are + # kept and everything else is dropped. When omitted, the entire document is kept. + # Examples: "article.main", "#content", "[role=main]". + include_selectors: nil, # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. @@ -399,10 +420,12 @@ module ContextDev sig do params( url: String, + exclude_selectors: T::Array[String], headers: T::Hash[Symbol, String], include_frames: T::Boolean, include_images: T::Boolean, include_links: T::Boolean, + include_selectors: T::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeMdParams::Pdf::OrHash, shorten_base64_images: T::Boolean, @@ -416,6 +439,10 @@ module ContextDev # Full URL to scrape into LLM usable Markdown (must include http:// or https:// # protocol) url:, + # CSS selectors to remove before conversion to Markdown. Applied after + # includeSelectors. Exclusion takes precedence: an element matching both is + # removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]". + exclude_selectors: nil, # Optional outbound HTTP headers forwarded only to the target URL, sent as # deep-object query params such as headers[X-Custom]=value. When provided, caching # is bypassed: the result is neither read from nor written to cache. @@ -426,6 +453,10 @@ module ContextDev include_images: nil, # Preserve hyperlinks in Markdown output include_links: nil, + # CSS selectors. When provided, only matching HTML subtrees (and their + # descendants) are kept before conversion to Markdown. When omitted, the entire + # document is kept. Examples: "article.main", "#content", "[role=main]". + include_selectors: nil, # Return a cached result if a prior scrape for the same parameters exists and is # younger than this many milliseconds. Defaults to 1 day (86400000 ms) when # omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh. diff --git a/sig/context_dev/models/web_web_crawl_md_params.rbs b/sig/context_dev/models/web_web_crawl_md_params.rbs index 8d39e42e..7f443035 100644 --- a/sig/context_dev/models/web_web_crawl_md_params.rbs +++ b/sig/context_dev/models/web_web_crawl_md_params.rbs @@ -3,10 +3,12 @@ module ContextDev type web_web_crawl_md_params = { url: String, + exclude_selectors: ::Array[String], follow_subdomains: bool, include_frames: bool, include_images: bool, include_links: bool, + include_selectors: ::Array[String], max_age_ms: Integer, max_depth: Integer, max_pages: Integer, @@ -26,6 +28,10 @@ module ContextDev attr_accessor url: String + attr_reader exclude_selectors: ::Array[String]? + + def exclude_selectors=: (::Array[String]) -> ::Array[String] + attr_reader follow_subdomains: bool? def follow_subdomains=: (bool) -> bool @@ -42,6 +48,10 @@ module ContextDev def include_links=: (bool) -> bool + attr_reader include_selectors: ::Array[String]? + + def include_selectors=: (::Array[String]) -> ::Array[String] + attr_reader max_age_ms: Integer? def max_age_ms=: (Integer) -> Integer @@ -86,10 +96,12 @@ module ContextDev def initialize: ( url: String, + ?exclude_selectors: ::Array[String], ?follow_subdomains: bool, ?include_frames: bool, ?include_images: bool, ?include_links: bool, + ?include_selectors: ::Array[String], ?max_age_ms: Integer, ?max_depth: Integer, ?max_pages: Integer, @@ -105,10 +117,12 @@ module ContextDev def to_hash: -> { url: String, + exclude_selectors: ::Array[String], follow_subdomains: bool, include_frames: bool, include_images: bool, include_links: bool, + include_selectors: ::Array[String], max_age_ms: Integer, max_depth: Integer, max_pages: Integer, diff --git a/sig/context_dev/models/web_web_scrape_html_params.rbs b/sig/context_dev/models/web_web_scrape_html_params.rbs index fdd0ca56..97c70452 100644 --- a/sig/context_dev/models/web_web_scrape_html_params.rbs +++ b/sig/context_dev/models/web_web_scrape_html_params.rbs @@ -3,8 +3,10 @@ module ContextDev type web_web_scrape_html_params = { url: String, + exclude_selectors: ::Array[String], headers: ::Hash[Symbol, String], include_frames: bool, + include_selectors: ::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeHTMLParams::Pdf, timeout_ms: Integer, @@ -18,6 +20,10 @@ module ContextDev attr_accessor url: String + attr_reader exclude_selectors: ::Array[String]? + + def exclude_selectors=: (::Array[String]) -> ::Array[String] + attr_reader headers: ::Hash[Symbol, String]? def headers=: (::Hash[Symbol, String]) -> ::Hash[Symbol, String] @@ -26,6 +32,10 @@ module ContextDev def include_frames=: (bool) -> bool + attr_reader include_selectors: ::Array[String]? + + def include_selectors=: (::Array[String]) -> ::Array[String] + attr_reader max_age_ms: Integer? def max_age_ms=: (Integer) -> Integer @@ -46,8 +56,10 @@ module ContextDev def initialize: ( url: String, + ?exclude_selectors: ::Array[String], ?headers: ::Hash[Symbol, String], ?include_frames: bool, + ?include_selectors: ::Array[String], ?max_age_ms: Integer, ?pdf: ContextDev::WebWebScrapeHTMLParams::Pdf, ?timeout_ms: Integer, @@ -57,8 +69,10 @@ module ContextDev def to_hash: -> { url: String, + exclude_selectors: ::Array[String], headers: ::Hash[Symbol, String], include_frames: bool, + include_selectors: ::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeHTMLParams::Pdf, timeout_ms: Integer, diff --git a/sig/context_dev/models/web_web_scrape_md_params.rbs b/sig/context_dev/models/web_web_scrape_md_params.rbs index b4f96dff..36975fb9 100644 --- a/sig/context_dev/models/web_web_scrape_md_params.rbs +++ b/sig/context_dev/models/web_web_scrape_md_params.rbs @@ -3,10 +3,12 @@ module ContextDev type web_web_scrape_md_params = { url: String, + exclude_selectors: ::Array[String], headers: ::Hash[Symbol, String], include_frames: bool, include_images: bool, include_links: bool, + include_selectors: ::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeMdParams::Pdf, :shorten_base64_images => bool, @@ -22,6 +24,10 @@ module ContextDev attr_accessor url: String + attr_reader exclude_selectors: ::Array[String]? + + def exclude_selectors=: (::Array[String]) -> ::Array[String] + attr_reader headers: ::Hash[Symbol, String]? def headers=: (::Hash[Symbol, String]) -> ::Hash[Symbol, String] @@ -38,6 +44,10 @@ module ContextDev def include_links=: (bool) -> bool + attr_reader include_selectors: ::Array[String]? + + def include_selectors=: (::Array[String]) -> ::Array[String] + attr_reader max_age_ms: Integer? def max_age_ms=: (Integer) -> Integer @@ -66,10 +76,12 @@ module ContextDev def initialize: ( url: String, + ?exclude_selectors: ::Array[String], ?headers: ::Hash[Symbol, String], ?include_frames: bool, ?include_images: bool, ?include_links: bool, + ?include_selectors: ::Array[String], ?max_age_ms: Integer, ?pdf: ContextDev::WebWebScrapeMdParams::Pdf, ?shorten_base64_images: bool, @@ -81,10 +93,12 @@ module ContextDev def to_hash: -> { url: String, + exclude_selectors: ::Array[String], headers: ::Hash[Symbol, String], include_frames: bool, include_images: bool, include_links: bool, + include_selectors: ::Array[String], max_age_ms: Integer, pdf: ContextDev::WebWebScrapeMdParams::Pdf, :shorten_base64_images => bool, diff --git a/sig/context_dev/resources/web.rbs b/sig/context_dev/resources/web.rbs index 35a753b8..9c73349e 100644 --- a/sig/context_dev/resources/web.rbs +++ b/sig/context_dev/resources/web.rbs @@ -65,10 +65,12 @@ module ContextDev def web_crawl_md: ( url: String, + ?exclude_selectors: ::Array[String], ?follow_subdomains: bool, ?include_frames: bool, ?include_images: bool, ?include_links: bool, + ?include_selectors: ::Array[String], ?max_age_ms: Integer, ?max_depth: Integer, ?max_pages: Integer, @@ -84,8 +86,10 @@ module ContextDev def web_scrape_html: ( url: String, + ?exclude_selectors: ::Array[String], ?headers: ::Hash[Symbol, String], ?include_frames: bool, + ?include_selectors: ::Array[String], ?max_age_ms: Integer, ?pdf: ContextDev::WebWebScrapeHTMLParams::Pdf, ?timeout_ms: Integer, @@ -105,10 +109,12 @@ module ContextDev def web_scrape_md: ( url: String, + ?exclude_selectors: ::Array[String], ?headers: ::Hash[Symbol, String], ?include_frames: bool, ?include_images: bool, ?include_links: bool, + ?include_selectors: ::Array[String], ?max_age_ms: Integer, ?pdf: ContextDev::WebWebScrapeMdParams::Pdf, ?shorten_base64_images: bool, From 13bbd3bc1543062b74ccffebe80518ea94652932 Mon Sep 17 00:00:00 2001 From: "stainless-app[bot]" <142633134+stainless-app[bot]@users.noreply.github.com> Date: Sun, 7 Jun 2026 22:03:32 +0000 Subject: [PATCH 2/2] release: 1.30.0 --- .release-please-manifest.json | 2 +- CHANGELOG.md | 8 ++++++++ Gemfile.lock | 2 +- README.md | 2 +- lib/context_dev/version.rb | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index b8af36c3..35c30adc 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.29.0" + ".": "1.30.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c6a8a7f..35d5e907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 1.30.0 (2026-06-07) + +Full Changelog: [v1.29.0...v1.30.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.29.0...v1.30.0) + +### Features + +* **api:** api update ([c897bbf](https://github.com/context-dot-dev/context-ruby-sdk/commit/c897bbf03235960c8036bd808c5239b935d94bf3)) + ## 1.29.0 (2026-06-07) Full Changelog: [v1.28.0...v1.29.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v1.28.0...v1.29.0) diff --git a/Gemfile.lock b/Gemfile.lock index c35ce637..e996b977 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT PATH remote: . specs: - context.dev (1.29.0) + context.dev (1.30.0) cgi connection_pool diff --git a/README.md b/README.md index 67ce9e28..8e49815d 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ To use this gem, install via Bundler by adding the following to your application ```ruby -gem "context.dev", "~> 1.29.0" +gem "context.dev", "~> 1.30.0" ``` diff --git a/lib/context_dev/version.rb b/lib/context_dev/version.rb index aa151347..03552d36 100644 --- a/lib/context_dev/version.rb +++ b/lib/context_dev/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ContextDev - VERSION = "1.29.0" + VERSION = "1.30.0" end