diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 511dd516..d1328ca9 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.6.0" + ".": "2.7.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 33b70809..3b484e38 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 37 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-960cb623c7ec84bf4dc0f5945cbc19eec9cca48271071f400d96066eaa55dbd6.yml -openapi_spec_hash: 84fd39e3f4dc964bf0c32d4e95da1b34 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-758c3a2fbd5b7be61c8e6e0ad2e6a5ec30695747bb675960d888c497647d13d7.yml +openapi_spec_hash: 00002d90bde02f67e174368ae470c597 config_hash: 2bea1743c84d63bd61f8501a6ea63065 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4beb8aba..086142b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 2.7.0 (2026-08-01) + +Full Changelog: [v2.6.0...v2.7.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v2.6.0...v2.7.0) + +### Features + +* **api:** api update ([b0bdc3e](https://github.com/context-dot-dev/context-ruby-sdk/commit/b0bdc3ec9322ce5854177cdaff808caaba427746)) +* **api:** api update ([17848b3](https://github.com/context-dot-dev/context-ruby-sdk/commit/17848b313dcbd098b21902bd2bd2cab6d9ad5ce2)) +* **api:** api update ([4db7386](https://github.com/context-dot-dev/context-ruby-sdk/commit/4db7386bf6f686a33beae7817ec9122e1adad47f)) + ## 2.6.0 (2026-07-31) Full Changelog: [v2.5.0...v2.6.0](https://github.com/context-dot-dev/context-ruby-sdk/compare/v2.5.0...v2.6.0) diff --git a/Gemfile.lock b/Gemfile.lock index 9d97d442..74451535 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT PATH remote: . specs: - context.dev (2.6.0) + context.dev (2.7.0) cgi connection_pool diff --git a/README.md b/README.md index ff7902e9..60b3e89a 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ To use this gem, install via Bundler by adding the following to your application ```ruby -gem "context.dev", "~> 2.6.0" +gem "context.dev", "~> 2.7.0" ``` diff --git a/lib/context_dev.rb b/lib/context_dev.rb index cf2e7604..924c7b02 100644 --- a/lib/context_dev.rb +++ b/lib/context_dev.rb @@ -70,12 +70,13 @@ require_relative "context_dev/models/brand_retrieve_response" require_relative "context_dev/models/brand_retrieve_simplified_params" require_relative "context_dev/models/brand_retrieve_simplified_response" -require_relative "context_dev/models/error" -require_relative "context_dev/models/error_count" +require_relative "context_dev/models/crawl_controls" +require_relative "context_dev/models/failure" require_relative "context_dev/models/industry_retrieve_naics_params" require_relative "context_dev/models/industry_retrieve_naics_response" require_relative "context_dev/models/industry_retrieve_sic_params" require_relative "context_dev/models/industry_retrieve_sic_response" +require_relative "context_dev/models/intake" require_relative "context_dev/models/monitor_create_params" require_relative "context_dev/models/monitor_create_response" require_relative "context_dev/models/monitor_delete_params" @@ -102,6 +103,7 @@ require_relative "context_dev/models/monitor_run_response" require_relative "context_dev/models/monitor_update_params" require_relative "context_dev/models/monitor_update_response" +require_relative "context_dev/models/page_error_count" require_relative "context_dev/models/parse_handle_params" require_relative "context_dev/models/parse_handle_response" require_relative "context_dev/models/utility_prefetch_params" diff --git a/lib/context_dev/models.rb b/lib/context_dev/models.rb index 2158b096..b4e42070 100644 --- a/lib/context_dev/models.rb +++ b/lib/context_dev/models.rb @@ -57,14 +57,16 @@ module ContextDev BrandRetrieveSimplifiedParams = ContextDev::Models::BrandRetrieveSimplifiedParams - Error = ContextDev::Models::Error + CrawlControls = ContextDev::Models::CrawlControls - ErrorCount = ContextDev::Models::ErrorCount + Failure = ContextDev::Models::Failure IndustryRetrieveNaicsParams = ContextDev::Models::IndustryRetrieveNaicsParams IndustryRetrieveSicParams = ContextDev::Models::IndustryRetrieveSicParams + Intake = ContextDev::Models::Intake + MonitorCreateParams = ContextDev::Models::MonitorCreateParams MonitorDeleteParams = ContextDev::Models::MonitorDeleteParams @@ -91,6 +93,8 @@ module ContextDev MonitorUpdateParams = ContextDev::Models::MonitorUpdateParams + PageErrorCount = ContextDev::Models::PageErrorCount + ParseHandleParams = ContextDev::Models::ParseHandleParams UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams diff --git a/lib/context_dev/models/batch_cancel_response.rb b/lib/context_dev/models/batch_cancel_response.rb index 6b92d64f..b05d3623 100644 --- a/lib/context_dev/models/batch_cancel_response.rb +++ b/lib/context_dev/models/batch_cancel_response.rb @@ -5,56 +5,57 @@ module Models # @see ContextDev::Resources::Batch#cancel class BatchCancelResponse < ContextDev::Internal::Type::BaseModel # @!attribute id - # Batch ID used to retrieve or cancel the job. + # Batch ID. # # @return [String] required :id, String + # @!attribute crawl + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + # + # @return [ContextDev::Models::CrawlControls, nil] + required :crawl, -> { ContextDev::CrawlControls }, nil?: true + # @!attribute credits - # Reserved and used credits. + # What this batch cost so far. # # @return [ContextDev::Models::BatchCancelResponse::Credits] required :credits, -> { ContextDev::Models::BatchCancelResponse::Credits } - # @!attribute error - # Why the batch failed. - # - # @return [ContextDev::Models::Error, nil] - required :error, -> { ContextDev::Error }, nil?: true - - # @!attribute errors - # Page failures grouped by error code. + # @!attribute format_ + # What each page is returned as. # - # @return [Array] - required :errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::ErrorCount] } + # @return [Symbol, ContextDev::Models::BatchCancelResponse::Format] + required :format_, enum: -> { ContextDev::Models::BatchCancelResponse::Format }, api_name: :format # @!attribute input - # Submission counts. + # What submission took in, and what it charged for. # - # @return [ContextDev::Models::BatchCancelResponse::Input] - required :input, -> { ContextDev::Models::BatchCancelResponse::Input } + # @return [ContextDev::Models::Intake] + required :input, -> { ContextDev::Intake } # @!attribute mode - # How pages are selected. + # How pages were selected. # # @return [Symbol, ContextDev::Models::BatchCancelResponse::Mode] required :mode, enum: -> { ContextDev::Models::BatchCancelResponse::Mode } + # @!attribute page_errors + # Page failures so far, grouped by error code and sorted by count. + # + # @return [Array] + required :page_errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::PageErrorCount] } + # @!attribute progress - # Current processing counts. Use `status` to check completion. + # How far the batch got before cancellation. # # @return [ContextDev::Models::BatchCancelResponse::Progress] required :progress, -> { ContextDev::Models::BatchCancelResponse::Progress } - # @!attribute results - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. - # - # @return [ContextDev::Models::BatchCancelResponse::Results, nil] - required :results, -> { ContextDev::Models::BatchCancelResponse::Results }, nil?: true - # @!attribute status - # Current state. `completed`, `cancelled`, and `failed` are final. + # Always `cancelling`. Work already in flight finishes; the batch reaches + # `cancelled` shortly after. # # @return [Symbol, ContextDev::Models::BatchCancelResponse::Status] required :status, enum: -> { ContextDev::Models::BatchCancelResponse::Status } @@ -66,113 +67,77 @@ class BatchCancelResponse < ContextDev::Internal::Type::BaseModel required :tags, ContextDev::Internal::Type::ArrayOf[String] # @!attribute timing + # There is no finish time yet — the batch is still winding down. # # @return [ContextDev::Models::BatchCancelResponse::Timing] required :timing, -> { ContextDev::Models::BatchCancelResponse::Timing } - # @!attribute type - # Output format. - # - # @return [Symbol, ContextDev::Models::BatchCancelResponse::Type] - required :type, enum: -> { ContextDev::Models::BatchCancelResponse::Type } - # @!attribute key_metadata # API key usage for this request. # # @return [ContextDev::Models::BatchCancelResponse::KeyMetadata, nil] optional :key_metadata, -> { ContextDev::Models::BatchCancelResponse::KeyMetadata } - # @!method initialize(id:, credits:, error:, errors:, input:, mode:, progress:, results:, status:, tags:, timing:, type:, key_metadata: nil) + # @!method initialize(id:, crawl:, credits:, format_:, input:, mode:, page_errors:, progress:, status:, tags:, timing:, key_metadata: nil) # Some parameter documentations has been truncated, see # {ContextDev::Models::BatchCancelResponse} for more details. # - # @param id [String] Batch ID used to retrieve or cancel the job. + # @param id [String] Batch ID. # - # @param credits [ContextDev::Models::BatchCancelResponse::Credits] Reserved and used credits. + # @param crawl [ContextDev::Models::CrawlControls, nil] The crawl controls as submitted, so the limits requested can be compared against # - # @param error [ContextDev::Models::Error, nil] Why the batch failed. + # @param credits [ContextDev::Models::BatchCancelResponse::Credits] What this batch cost so far. # - # @param errors [Array] Page failures grouped by error code. + # @param format_ [Symbol, ContextDev::Models::BatchCancelResponse::Format] What each page is returned as. # - # @param input [ContextDev::Models::BatchCancelResponse::Input] Submission counts. + # @param input [ContextDev::Models::Intake] What submission took in, and what it charged for. # - # @param mode [Symbol, ContextDev::Models::BatchCancelResponse::Mode] How pages are selected. + # @param mode [Symbol, ContextDev::Models::BatchCancelResponse::Mode] How pages were selected. # - # @param progress [ContextDev::Models::BatchCancelResponse::Progress] Current processing counts. Use `status` to check completion. + # @param page_errors [Array] Page failures so far, grouped by error code and sorted by count. # - # @param results [ContextDev::Models::BatchCancelResponse::Results, nil] Download links available when the batch finishes. GET /batch/{batch_id}/results + # @param progress [ContextDev::Models::BatchCancelResponse::Progress] How far the batch got before cancellation. # - # @param status [Symbol, ContextDev::Models::BatchCancelResponse::Status] Current state. `completed`, `cancelled`, and `failed` are final. + # @param status [Symbol, ContextDev::Models::BatchCancelResponse::Status] Always `cancelling`. Work already in flight finishes; the batch reaches `cancell # # @param tags [Array] Tags stored on the batch at submission. # - # @param timing [ContextDev::Models::BatchCancelResponse::Timing] - # - # @param type [Symbol, ContextDev::Models::BatchCancelResponse::Type] Output format. + # @param timing [ContextDev::Models::BatchCancelResponse::Timing] There is no finish time yet — the batch is still winding down. # # @param key_metadata [ContextDev::Models::BatchCancelResponse::KeyMetadata] API key usage for this request. # @see ContextDev::Models::BatchCancelResponse#credits class Credits < ContextDev::Internal::Type::BaseModel - # @!attribute charged - # Credits used by successful pages. + # @!attribute reserved + # Credits debited at submission. The unspent remainder is refunded once the batch + # settles — read `credits.refunded` from GET /batch/{batch_id} then. # # @return [Integer] - required :charged, Integer + required :reserved, Integer - # @!attribute estimated - # Credits reserved when the batch was accepted. - # - # @return [Integer] - required :estimated, Integer - - # @!method initialize(charged:, estimated:) - # Reserved and used credits. + # @!method initialize(reserved:) + # Some parameter documentations has been truncated, see + # {ContextDev::Models::BatchCancelResponse::Credits} for more details. # - # @param charged [Integer] Credits used by successful pages. + # What this batch cost so far. # - # @param estimated [Integer] Credits reserved when the batch was accepted. + # @param reserved [Integer] Credits debited at submission. The unspent remainder is refunded once the batch end - # @see ContextDev::Models::BatchCancelResponse#input - class Input < ContextDev::Internal::Type::BaseModel - # @!attribute accepted - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - # - # @return [Integer] - required :accepted, Integer - - # @!attribute duplicates - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - # - # @return [Integer] - required :duplicates, Integer - - # @!attribute invalid - # Pages rejected during validation. - # - # @return [Integer] - required :invalid, Integer + # What each page is returned as. + # + # @see ContextDev::Models::BatchCancelResponse#format_ + module Format + extend ContextDev::Internal::Type::Enum - # @!attribute submitted - # Pages submitted before validation. For a crawl, the page limit. - # - # @return [Integer] - required :submitted, Integer + MARKDOWN = :markdown + HTML = :html - # @!method initialize(accepted:, duplicates:, invalid:, submitted:) - # Submission counts. - # - # @param accepted [Integer] Pages accepted, or the crawl page limit. Credits are reserved for this count. - # - # @param duplicates [Integer] Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - # - # @param invalid [Integer] Pages rejected during validation. - # - # @param submitted [Integer] Pages submitted before validation. For a crawl, the page limit. + # @!method self.values + # @return [Array] end - # How pages are selected. + # How pages were selected. # # @see ContextDev::Models::BatchCancelResponse#mode module Mode @@ -188,100 +153,41 @@ module Mode # @see ContextDev::Models::BatchCancelResponse#progress class Progress < ContextDev::Internal::Type::BaseModel # @!attribute failed - # Pages that could not be scraped. + # Pages that could not be scraped before the request landed. # # @return [Integer] required :failed, Integer # @!attribute pending - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages that will now be skipped, and refunded when the batch settles. # # @return [Integer] required :pending, Integer # @!attribute succeeded - # Pages scraped successfully. + # Pages scraped successfully before the request landed. # # @return [Integer] required :succeeded, Integer # @!method initialize(failed:, pending:, succeeded:) - # Some parameter documentations has been truncated, see - # {ContextDev::Models::BatchCancelResponse::Progress} for more details. + # How far the batch got before cancellation. # - # Current processing counts. Use `status` to check completion. + # @param failed [Integer] Pages that could not be scraped before the request landed. # - # @param failed [Integer] Pages that could not be scraped. + # @param pending [Integer] Reserved pages that will now be skipped, and refunded when the batch settles. # - # @param pending [Integer] Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # - # @param succeeded [Integer] Pages scraped successfully. + # @param succeeded [Integer] Pages scraped successfully before the request landed. end - # @see ContextDev::Models::BatchCancelResponse#results - class Results < ContextDev::Internal::Type::BaseModel - # @!attribute expires_at - # When the download URLs expire. - # - # @return [String] - required :expires_at, String - - # @!attribute files - # Result files. Order is not guaranteed. - # - # @return [Array] - required :files, - -> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchCancelResponse::Results::File] } - - # @!method initialize(expires_at:, files:) - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. - # - # @param expires_at [String] When the download URLs expire. - # - # @param files [Array] Result files. Order is not guaranteed. - - class File < ContextDev::Internal::Type::BaseModel - # @!attribute bytes - # Compressed file size in bytes. - # - # @return [Integer] - required :bytes, Integer - - # @!attribute items - # Results in this file. - # - # @return [Integer] - required :items, Integer - - # @!attribute url - # Temporary URL for a gzipped NDJSON file. - # - # @return [String] - required :url, String - - # @!method initialize(bytes:, items:, url:) - # @param bytes [Integer] Compressed file size in bytes. - # - # @param items [Integer] Results in this file. - # - # @param url [String] Temporary URL for a gzipped NDJSON file. - end - end - - # Current state. `completed`, `cancelled`, and `failed` are final. + # Always `cancelling`. Work already in flight finishes; the batch reaches + # `cancelled` shortly after. # # @see ContextDev::Models::BatchCancelResponse#status module Status extend ContextDev::Internal::Type::Enum - QUEUED = :queued - RUNNING = :running CANCELLING = :cancelling - COMPLETED = :completed - CANCELLED = :cancelled - FAILED = :failed # @!method self.values # @return [Array] @@ -289,12 +195,6 @@ module Status # @see ContextDev::Models::BatchCancelResponse#timing class Timing < ContextDev::Internal::Type::BaseModel - # @!attribute completed_at - # When processing finished. Null while active. - # - # @return [String, nil] - required :completed_at, String, nil?: true - # @!attribute created_at # When the batch was created. # @@ -302,30 +202,17 @@ class Timing < ContextDev::Internal::Type::BaseModel required :created_at, String # @!attribute started_at - # When processing started. Null while queued. + # When processing started. Null if it was cancelled while still queued. # # @return [String, nil] required :started_at, String, nil?: true - # @!method initialize(completed_at:, created_at:, started_at:) - # @param completed_at [String, nil] When processing finished. Null while active. + # @!method initialize(created_at:, started_at:) + # There is no finish time yet — the batch is still winding down. # # @param created_at [String] When the batch was created. # - # @param started_at [String, nil] When processing started. Null while queued. - end - - # Output format. - # - # @see ContextDev::Models::BatchCancelResponse#type - module Type - extend ContextDev::Internal::Type::Enum - - MARKDOWN = :markdown - HTML = :html - - # @!method self.values - # @return [Array] + # @param started_at [String, nil] When processing started. Null if it was cancelled while still queued. end # @see ContextDev::Models::BatchCancelResponse#key_metadata diff --git a/lib/context_dev/models/batch_list_response.rb b/lib/context_dev/models/batch_list_response.rb index 94429714..c6b7c572 100644 --- a/lib/context_dev/models/batch_list_response.rb +++ b/lib/context_dev/models/batch_list_response.rb @@ -48,45 +48,61 @@ class Data < ContextDev::Internal::Type::BaseModel # @return [String] required :id, String + # @!attribute crawl + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + # + # @return [ContextDev::Models::CrawlControls, nil] + required :crawl, -> { ContextDev::CrawlControls }, nil?: true + # @!attribute credits - # Reserved and used credits. + # What this batch has done to your credit balance. # # @return [ContextDev::Models::BatchListResponse::Data::Credits] required :credits, -> { ContextDev::Models::BatchListResponse::Data::Credits } - # @!attribute error - # Why the batch failed. + # @!attribute failure + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. # - # @return [ContextDev::Models::Error, nil] - required :error, -> { ContextDev::Error }, nil?: true + # @return [ContextDev::Models::Failure, nil] + required :failure, -> { ContextDev::Failure }, nil?: true - # @!attribute errors - # Page failures grouped by error code. + # @!attribute format_ + # What each page is returned as. Matches `input.data.format` on the submit + # request. # - # @return [Array] - required :errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::ErrorCount] } + # @return [Symbol, ContextDev::Models::BatchListResponse::Data::Format] + required :format_, enum: -> { ContextDev::Models::BatchListResponse::Data::Format }, api_name: :format # @!attribute input - # Submission counts. + # What submission took in, and what it charged for. # - # @return [ContextDev::Models::BatchListResponse::Data::Input] - required :input, -> { ContextDev::Models::BatchListResponse::Data::Input } + # @return [ContextDev::Models::Intake] + required :input, -> { ContextDev::Intake } # @!attribute mode - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. # # @return [Symbol, ContextDev::Models::BatchListResponse::Data::Mode] required :mode, enum: -> { ContextDev::Models::BatchListResponse::Data::Mode } + # @!attribute page_errors + # Individual page failures grouped by error code, sorted by count. Unrelated to + # `failure`, which is the batch itself failing. + # + # @return [Array] + required :page_errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::PageErrorCount] } + # @!attribute progress - # Current processing counts. Use `status` to check completion. + # Pages attempted so far. Use `status` to check completion. # # @return [ContextDev::Models::BatchListResponse::Data::Progress] required :progress, -> { ContextDev::Models::BatchListResponse::Data::Progress } # @!attribute results - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. # # @return [ContextDev::Models::BatchListResponse::Data::Results, nil] required :results, -> { ContextDev::Models::BatchListResponse::Data::Results }, nil?: true @@ -108,13 +124,7 @@ class Data < ContextDev::Internal::Type::BaseModel # @return [ContextDev::Models::BatchListResponse::Data::Timing] required :timing, -> { ContextDev::Models::BatchListResponse::Data::Timing } - # @!attribute type - # Output format. - # - # @return [Symbol, ContextDev::Models::BatchListResponse::Data::Type] - required :type, enum: -> { ContextDev::Models::BatchListResponse::Data::Type } - - # @!method initialize(id:, credits:, error:, errors:, input:, mode:, progress:, results:, status:, tags:, timing:, type:) + # @!method initialize(id:, crawl:, credits:, failure:, format_:, input:, mode:, page_errors:, progress:, results:, status:, tags:, timing:) # Some parameter documentations has been truncated, see # {ContextDev::Models::BatchListResponse::Data} for more details. # @@ -122,89 +132,82 @@ class Data < ContextDev::Internal::Type::BaseModel # # @param id [String] Batch ID used to retrieve or cancel the job. # - # @param credits [ContextDev::Models::BatchListResponse::Data::Credits] Reserved and used credits. + # @param crawl [ContextDev::Models::CrawlControls, nil] The crawl controls as submitted, so the limits requested can be compared against # - # @param error [ContextDev::Models::Error, nil] Why the batch failed. + # @param credits [ContextDev::Models::BatchListResponse::Data::Credits] What this batch has done to your credit balance. # - # @param errors [Array] Page failures grouped by error code. + # @param failure [ContextDev::Models::Failure, nil] A failure of the batch as a whole, distinct from the per-page failures in + # `page\_ # - # @param input [ContextDev::Models::BatchListResponse::Data::Input] Submission counts. + # @param format_ [Symbol, ContextDev::Models::BatchListResponse::Data::Format] What each page is returned as. Matches `input.data.format` on the submit request # - # @param mode [Symbol, ContextDev::Models::BatchListResponse::Data::Mode] How pages are selected. + # @param input [ContextDev::Models::Intake] What submission took in, and what it charged for. # - # @param progress [ContextDev::Models::BatchListResponse::Data::Progress] Current processing counts. Use `status` to check completion. + # @param mode [Symbol, ContextDev::Models::BatchListResponse::Data::Mode] How pages were selected. Matches `input.mode` on the submit request. # - # @param results [ContextDev::Models::BatchListResponse::Data::Results, nil] Download links available when the batch finishes. GET /batch/{batch_id}/results + # @param page_errors [Array] Individual page failures grouped by error code, sorted by count. Unrelated to `f + # + # @param progress [ContextDev::Models::BatchListResponse::Data::Progress] Pages attempted so far. Use `status` to check completion. + # + # @param results [ContextDev::Models::BatchListResponse::Data::Results, nil] Download links, available once the batch reaches a final status and null before # # @param status [Symbol, ContextDev::Models::BatchListResponse::Data::Status] Current state. `completed`, `cancelled`, and `failed` are final. # # @param tags [Array] Tags stored on the batch at submission. # # @param timing [ContextDev::Models::BatchListResponse::Data::Timing] - # - # @param type [Symbol, ContextDev::Models::BatchListResponse::Data::Type] Output format. # @see ContextDev::Models::BatchListResponse::Data#credits class Credits < ContextDev::Internal::Type::BaseModel - # @!attribute charged - # Credits used by successful pages. - # - # @return [Integer] - required :charged, Integer - - # @!attribute estimated - # Credits reserved when the batch was accepted. - # - # @return [Integer] - required :estimated, Integer - - # @!method initialize(charged:, estimated:) - # Reserved and used credits. - # - # @param charged [Integer] Credits used by successful pages. - # - # @param estimated [Integer] Credits reserved when the batch was accepted. - end - - # @see ContextDev::Models::BatchListResponse::Data#input - class Input < ContextDev::Internal::Type::BaseModel - # @!attribute accepted - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - # - # @return [Integer] - required :accepted, Integer - - # @!attribute duplicates - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. + # @!attribute net + # `reserved` minus `refunded` — what the batch has cost so far. Equal to + # `reserved` until the batch settles. # # @return [Integer] - required :duplicates, Integer + required :net, Integer - # @!attribute invalid - # Pages rejected during validation. + # @!attribute refunded + # Credits returned for pages that did not succeed. Stays 0 until the batch reaches + # a final status, then settles in one movement. # # @return [Integer] - required :invalid, Integer + required :refunded, Integer - # @!attribute submitted - # Pages submitted before validation. For a crawl, the page limit. + # @!attribute reserved + # Credits debited from your balance the moment the batch was accepted. This is a + # charge, not a forecast — the whole amount leaves the balance up front. # # @return [Integer] - required :submitted, Integer + required :reserved, Integer - # @!method initialize(accepted:, duplicates:, invalid:, submitted:) - # Submission counts. + # @!method initialize(net:, refunded:, reserved:) + # Some parameter documentations has been truncated, see + # {ContextDev::Models::BatchListResponse::Data::Credits} for more details. # - # @param accepted [Integer] Pages accepted, or the crawl page limit. Credits are reserved for this count. + # What this batch has done to your credit balance. # - # @param duplicates [Integer] Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. + # @param net [Integer] `reserved` minus `refunded` — what the batch has cost so far. Equal to `reserved # - # @param invalid [Integer] Pages rejected during validation. + # @param refunded [Integer] Credits returned for pages that did not succeed. Stays 0 until the batch reaches # - # @param submitted [Integer] Pages submitted before validation. For a crawl, the page limit. + # @param reserved [Integer] Credits debited from your balance the moment the batch was accepted. This is a c + end + + # What each page is returned as. Matches `input.data.format` on the submit + # request. + # + # @see ContextDev::Models::BatchListResponse::Data#format_ + module Format + extend ContextDev::Internal::Type::Enum + + MARKDOWN = :markdown + HTML = :html + + # @!method self.values + # @return [Array] end - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. # # @see ContextDev::Models::BatchListResponse::Data#mode module Mode @@ -226,8 +229,9 @@ class Progress < ContextDev::Internal::Type::BaseModel required :failed, Integer # @!attribute pending - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + # never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + # final, because its unspent budget was never real pages. # # @return [Integer] required :pending, Integer @@ -242,11 +246,11 @@ class Progress < ContextDev::Internal::Type::BaseModel # Some parameter documentations has been truncated, see # {ContextDev::Models::BatchListResponse::Data::Progress} for more details. # - # Current processing counts. Use `status` to check completion. + # Pages attempted so far. Use `status` to check completion. # # @param failed [Integer] Pages that could not be scraped. # - # @param pending [Integer] Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can + # @param pending [Integer] Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it # # @param succeeded [Integer] Pages scraped successfully. end @@ -267,8 +271,8 @@ class Results < ContextDev::Internal::Type::BaseModel -> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchListResponse::Data::Results::File] } # @!method initialize(expires_at:, files:) - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. # # @param expires_at [String] When the download URLs expire. # @@ -346,19 +350,6 @@ class Timing < ContextDev::Internal::Type::BaseModel # # @param started_at [String, nil] When processing started. Null while queued. end - - # Output format. - # - # @see ContextDev::Models::BatchListResponse::Data#type - module Type - extend ContextDev::Internal::Type::Enum - - MARKDOWN = :markdown - HTML = :html - - # @!method self.values - # @return [Array] - end end # @see ContextDev::Models::BatchListResponse#key_metadata diff --git a/lib/context_dev/models/batch_retrieve_response.rb b/lib/context_dev/models/batch_retrieve_response.rb index 7f094b23..1ad0da1e 100644 --- a/lib/context_dev/models/batch_retrieve_response.rb +++ b/lib/context_dev/models/batch_retrieve_response.rb @@ -10,29 +10,38 @@ class BatchRetrieveResponse < ContextDev::Internal::Type::BaseModel # @return [String] required :id, String + # @!attribute crawl + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + # + # @return [ContextDev::Models::CrawlControls, nil] + required :crawl, -> { ContextDev::CrawlControls }, nil?: true + # @!attribute credits - # Reserved and used credits. + # What this batch has done to your credit balance. # # @return [ContextDev::Models::BatchRetrieveResponse::Credits] required :credits, -> { ContextDev::Models::BatchRetrieveResponse::Credits } - # @!attribute error - # Why the batch failed. + # @!attribute failure + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. # - # @return [ContextDev::Models::Error, nil] - required :error, -> { ContextDev::Error }, nil?: true + # @return [ContextDev::Models::Failure, nil] + required :failure, -> { ContextDev::Failure }, nil?: true - # @!attribute errors - # Page failures grouped by error code. + # @!attribute format_ + # What each page is returned as. Matches `input.data.format` on the submit + # request. # - # @return [Array] - required :errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::ErrorCount] } + # @return [Symbol, ContextDev::Models::BatchRetrieveResponse::Format] + required :format_, enum: -> { ContextDev::Models::BatchRetrieveResponse::Format }, api_name: :format # @!attribute input - # Submission counts. + # What submission took in, and what it charged for. # - # @return [ContextDev::Models::BatchRetrieveResponse::Input] - required :input, -> { ContextDev::Models::BatchRetrieveResponse::Input } + # @return [ContextDev::Models::Intake] + required :input, -> { ContextDev::Intake } # @!attribute invalid_urls # Rejected URLs, up to 100. These are not charged. @@ -42,20 +51,27 @@ class BatchRetrieveResponse < ContextDev::Internal::Type::BaseModel -> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchRetrieveResponse::InvalidURL] } # @!attribute mode - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. # # @return [Symbol, ContextDev::Models::BatchRetrieveResponse::Mode] required :mode, enum: -> { ContextDev::Models::BatchRetrieveResponse::Mode } + # @!attribute page_errors + # Individual page failures grouped by error code, sorted by count. Unrelated to + # `failure`, which is the batch itself failing. + # + # @return [Array] + required :page_errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::PageErrorCount] } + # @!attribute progress - # Current processing counts. Use `status` to check completion. + # Pages attempted so far. Use `status` to check completion. # # @return [ContextDev::Models::BatchRetrieveResponse::Progress] required :progress, -> { ContextDev::Models::BatchRetrieveResponse::Progress } # @!attribute results - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. # # @return [ContextDev::Models::BatchRetrieveResponse::Results, nil] required :results, -> { ContextDev::Models::BatchRetrieveResponse::Results }, nil?: true @@ -77,45 +93,38 @@ class BatchRetrieveResponse < ContextDev::Internal::Type::BaseModel # @return [ContextDev::Models::BatchRetrieveResponse::Timing] required :timing, -> { ContextDev::Models::BatchRetrieveResponse::Timing } - # @!attribute type - # Output format. - # - # @return [Symbol, ContextDev::Models::BatchRetrieveResponse::Type] - required :type, enum: -> { ContextDev::Models::BatchRetrieveResponse::Type } - # @!attribute key_metadata # API key usage for this request. # # @return [ContextDev::Models::BatchRetrieveResponse::KeyMetadata, nil] optional :key_metadata, -> { ContextDev::Models::BatchRetrieveResponse::KeyMetadata } - # @!attribute webhook_secret - # Webhook signing secret. Also returned by GET /batch/{batch_id}. - # - # @return [String, nil] - optional :webhook_secret, String - - # @!method initialize(id:, credits:, error:, errors:, input:, invalid_urls:, mode:, progress:, results:, status:, tags:, timing:, type:, key_metadata: nil, webhook_secret: nil) + # @!method initialize(id:, crawl:, credits:, failure:, format_:, input:, invalid_urls:, mode:, page_errors:, progress:, results:, status:, tags:, timing:, key_metadata: nil) # Some parameter documentations has been truncated, see # {ContextDev::Models::BatchRetrieveResponse} for more details. # # @param id [String] Batch ID used to retrieve or cancel the job. # - # @param credits [ContextDev::Models::BatchRetrieveResponse::Credits] Reserved and used credits. + # @param crawl [ContextDev::Models::CrawlControls, nil] The crawl controls as submitted, so the limits requested can be compared against + # + # @param credits [ContextDev::Models::BatchRetrieveResponse::Credits] What this batch has done to your credit balance. # - # @param error [ContextDev::Models::Error, nil] Why the batch failed. + # @param failure [ContextDev::Models::Failure, nil] A failure of the batch as a whole, distinct from the per-page failures in + # `page\_ # - # @param errors [Array] Page failures grouped by error code. + # @param format_ [Symbol, ContextDev::Models::BatchRetrieveResponse::Format] What each page is returned as. Matches `input.data.format` on the submit request # - # @param input [ContextDev::Models::BatchRetrieveResponse::Input] Submission counts. + # @param input [ContextDev::Models::Intake] What submission took in, and what it charged for. # # @param invalid_urls [Array] Rejected URLs, up to 100. These are not charged. # - # @param mode [Symbol, ContextDev::Models::BatchRetrieveResponse::Mode] How pages are selected. + # @param mode [Symbol, ContextDev::Models::BatchRetrieveResponse::Mode] How pages were selected. Matches `input.mode` on the submit request. # - # @param progress [ContextDev::Models::BatchRetrieveResponse::Progress] Current processing counts. Use `status` to check completion. + # @param page_errors [Array] Individual page failures grouped by error code, sorted by count. Unrelated to `f # - # @param results [ContextDev::Models::BatchRetrieveResponse::Results, nil] Download links available when the batch finishes. GET /batch/{batch_id}/results + # @param progress [ContextDev::Models::BatchRetrieveResponse::Progress] Pages attempted so far. Use `status` to check completion. + # + # @param results [ContextDev::Models::BatchRetrieveResponse::Results, nil] Download links, available once the batch reaches a final status and null before # # @param status [Symbol, ContextDev::Models::BatchRetrieveResponse::Status] Current state. `completed`, `cancelled`, and `failed` are final. # @@ -123,70 +132,56 @@ class BatchRetrieveResponse < ContextDev::Internal::Type::BaseModel # # @param timing [ContextDev::Models::BatchRetrieveResponse::Timing] # - # @param type [Symbol, ContextDev::Models::BatchRetrieveResponse::Type] Output format. - # # @param key_metadata [ContextDev::Models::BatchRetrieveResponse::KeyMetadata] API key usage for this request. - # - # @param webhook_secret [String] Webhook signing secret. Also returned by GET /batch/{batch_id}. # @see ContextDev::Models::BatchRetrieveResponse#credits class Credits < ContextDev::Internal::Type::BaseModel - # @!attribute charged - # Credits used by successful pages. - # - # @return [Integer] - required :charged, Integer - - # @!attribute estimated - # Credits reserved when the batch was accepted. - # - # @return [Integer] - required :estimated, Integer - - # @!method initialize(charged:, estimated:) - # Reserved and used credits. - # - # @param charged [Integer] Credits used by successful pages. - # - # @param estimated [Integer] Credits reserved when the batch was accepted. - end - - # @see ContextDev::Models::BatchRetrieveResponse#input - class Input < ContextDev::Internal::Type::BaseModel - # @!attribute accepted - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - # - # @return [Integer] - required :accepted, Integer - - # @!attribute duplicates - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. + # @!attribute net + # `reserved` minus `refunded` — what the batch has cost so far. Equal to + # `reserved` until the batch settles. # # @return [Integer] - required :duplicates, Integer + required :net, Integer - # @!attribute invalid - # Pages rejected during validation. + # @!attribute refunded + # Credits returned for pages that did not succeed. Stays 0 until the batch reaches + # a final status, then settles in one movement. # # @return [Integer] - required :invalid, Integer + required :refunded, Integer - # @!attribute submitted - # Pages submitted before validation. For a crawl, the page limit. + # @!attribute reserved + # Credits debited from your balance the moment the batch was accepted. This is a + # charge, not a forecast — the whole amount leaves the balance up front. # # @return [Integer] - required :submitted, Integer + required :reserved, Integer - # @!method initialize(accepted:, duplicates:, invalid:, submitted:) - # Submission counts. + # @!method initialize(net:, refunded:, reserved:) + # Some parameter documentations has been truncated, see + # {ContextDev::Models::BatchRetrieveResponse::Credits} for more details. # - # @param accepted [Integer] Pages accepted, or the crawl page limit. Credits are reserved for this count. + # What this batch has done to your credit balance. # - # @param duplicates [Integer] Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. + # @param net [Integer] `reserved` minus `refunded` — what the batch has cost so far. Equal to `reserved # - # @param invalid [Integer] Pages rejected during validation. + # @param refunded [Integer] Credits returned for pages that did not succeed. Stays 0 until the batch reaches # - # @param submitted [Integer] Pages submitted before validation. For a crawl, the page limit. + # @param reserved [Integer] Credits debited from your balance the moment the batch was accepted. This is a c + end + + # What each page is returned as. Matches `input.data.format` on the submit + # request. + # + # @see ContextDev::Models::BatchRetrieveResponse#format_ + module Format + extend ContextDev::Internal::Type::Enum + + MARKDOWN = :markdown + HTML = :html + + # @!method self.values + # @return [Array] end class InvalidURL < ContextDev::Internal::Type::BaseModel @@ -208,7 +203,7 @@ class InvalidURL < ContextDev::Internal::Type::BaseModel # @param url [String] Rejected URL. end - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. # # @see ContextDev::Models::BatchRetrieveResponse#mode module Mode @@ -230,8 +225,9 @@ class Progress < ContextDev::Internal::Type::BaseModel required :failed, Integer # @!attribute pending - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + # never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + # final, because its unspent budget was never real pages. # # @return [Integer] required :pending, Integer @@ -246,11 +242,11 @@ class Progress < ContextDev::Internal::Type::BaseModel # Some parameter documentations has been truncated, see # {ContextDev::Models::BatchRetrieveResponse::Progress} for more details. # - # Current processing counts. Use `status` to check completion. + # Pages attempted so far. Use `status` to check completion. # # @param failed [Integer] Pages that could not be scraped. # - # @param pending [Integer] Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can + # @param pending [Integer] Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it # # @param succeeded [Integer] Pages scraped successfully. end @@ -271,8 +267,8 @@ class Results < ContextDev::Internal::Type::BaseModel -> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchRetrieveResponse::Results::File] } # @!method initialize(expires_at:, files:) - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. # # @param expires_at [String] When the download URLs expire. # @@ -351,19 +347,6 @@ class Timing < ContextDev::Internal::Type::BaseModel # @param started_at [String, nil] When processing started. Null while queued. end - # Output format. - # - # @see ContextDev::Models::BatchRetrieveResponse#type - module Type - extend ContextDev::Internal::Type::Enum - - MARKDOWN = :markdown - HTML = :html - - # @!method self.values - # @return [Array] - end - # @see ContextDev::Models::BatchRetrieveResponse#key_metadata class KeyMetadata < ContextDev::Internal::Type::BaseModel # @!attribute credits_consumed diff --git a/lib/context_dev/models/crawl_controls.rb b/lib/context_dev/models/crawl_controls.rb new file mode 100644 index 00000000..2354f042 --- /dev/null +++ b/lib/context_dev/models/crawl_controls.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +module ContextDev + module Models + class CrawlControls < ContextDev::Internal::Type::BaseModel + # @!attribute follow_subdomains + # Whether links to subdomains were followed. Always false for a sitemap crawl. + # + # @return [Boolean] + required :follow_subdomains, ContextDev::Internal::Type::Boolean + + # @!attribute max_depth + # Link depth limit. Always 0 for a sitemap crawl, which never follows links off + # its URLs; null when a `start_url` crawl set no limit. + # + # @return [Integer, nil] + required :max_depth, Integer, nil?: true + + # @!attribute max_pages + # The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs + # its sitemap actually lists, up to this many, so `input.reserved` is often lower. + # + # @return [Integer] + required :max_pages, Integer + + # @!attribute source + # Where the crawl started. + # + # @return [ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1] + required :source, union: -> { ContextDev::CrawlControls::Source } + + # @!attribute url_pattern + # RE2 pattern URLs had to match to be crawled. Null when the crawl set none. + # + # @return [String, nil] + required :url_pattern, String, nil?: true + + # @!method initialize(follow_subdomains:, max_depth:, max_pages:, source:, url_pattern:) + # Some parameter documentations has been truncated, see + # {ContextDev::Models::CrawlControls} for more details. + # + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + # + # @param follow_subdomains [Boolean] Whether links to subdomains were followed. Always false for a sitemap crawl. + # + # @param max_depth [Integer, nil] Link depth limit. Always 0 for a sitemap crawl, which never follows links off it + # + # @param max_pages [Integer] The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs it + # + # @param source [ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1] Where the crawl started. + # + # @param url_pattern [String, nil] RE2 pattern URLs had to match to be crawled. Null when the crawl set none. + + # Where the crawl started. + # + # @see ContextDev::Models::CrawlControls#source + module Source + extend ContextDev::Internal::Type::Union + + variant -> { ContextDev::CrawlControls::Source::UnionMember0 } + + variant -> { ContextDev::CrawlControls::Source::UnionMember1 } + + class UnionMember0 < ContextDev::Internal::Type::BaseModel + # @!attribute type + # + # @return [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember0::Type] + required :type, enum: -> { ContextDev::CrawlControls::Source::UnionMember0::Type } + + # @!attribute url + # Page the crawl started from. + # + # @return [String] + required :url, String + + # @!method initialize(type:, url:) + # @param type [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember0::Type] + # + # @param url [String] Page the crawl started from. + + # @see ContextDev::Models::CrawlControls::Source::UnionMember0#type + module Type + extend ContextDev::Internal::Type::Enum + + START_URL = :start_url + + # @!method self.values + # @return [Array] + end + end + + class UnionMember1 < ContextDev::Internal::Type::BaseModel + # @!attribute domain + # Domain whose sitemap supplied the pages. + # + # @return [String] + required :domain, String + + # @!attribute type + # + # @return [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember1::Type] + required :type, enum: -> { ContextDev::CrawlControls::Source::UnionMember1::Type } + + # @!method initialize(domain:, type:) + # @param domain [String] Domain whose sitemap supplied the pages. + # + # @param type [Symbol, ContextDev::Models::CrawlControls::Source::UnionMember1::Type] + + # @see ContextDev::Models::CrawlControls::Source::UnionMember1#type + module Type + extend ContextDev::Internal::Type::Enum + + SITEMAP = :sitemap + + # @!method self.values + # @return [Array] + end + end + + # @!method self.variants + # @return [Array(ContextDev::Models::CrawlControls::Source::UnionMember0, ContextDev::Models::CrawlControls::Source::UnionMember1)] + end + end + end +end diff --git a/lib/context_dev/models/error.rb b/lib/context_dev/models/error.rb deleted file mode 100644 index 3e5e2c13..00000000 --- a/lib/context_dev/models/error.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module ContextDev - module Models - class Error < ContextDev::Internal::Type::BaseModel - # @!attribute code - # Batch error code. - # - # @return [String] - required :code, String - - # @!attribute message - # Batch error message. - # - # @return [String] - required :message, String - - # @!method initialize(code:, message:) - # Why the batch failed. - # - # @param code [String] Batch error code. - # - # @param message [String] Batch error message. - end - end -end diff --git a/lib/context_dev/models/failure.rb b/lib/context_dev/models/failure.rb new file mode 100644 index 00000000..3bad930a --- /dev/null +++ b/lib/context_dev/models/failure.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module ContextDev + module Models + class Failure < ContextDev::Internal::Type::BaseModel + # @!attribute code + # Why the batch itself stopped. + # + # @return [String] + required :code, String + + # @!attribute message + # Human-readable explanation. + # + # @return [String] + required :message, String + + # @!method initialize(code:, message:) + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. + # + # @param code [String] Why the batch itself stopped. + # + # @param message [String] Human-readable explanation. + end + end +end diff --git a/lib/context_dev/models/intake.rb b/lib/context_dev/models/intake.rb new file mode 100644 index 00000000..01411417 --- /dev/null +++ b/lib/context_dev/models/intake.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module ContextDev + module Models + class Intake < ContextDev::Internal::Type::BaseModel + # @!attribute duplicates + # URLs dropped before reserving because another entry resolved to the same page. + # Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than + # once. + # + # @return [Integer] + required :duplicates, Integer + + # @!attribute invalid + # URLs from your list rejected as unusable; the same ones are itemised in + # `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable + # page is rejected outright with a 400 rather than accepted with an empty list. + # + # @return [Integer, nil] + required :invalid, Integer, nil?: true + + # @!attribute reserved + # Pages credits were reserved for. Everything else — progress, the refund, the + # completion percentage — is measured against this. + # + # @return [Integer] + required :reserved, Integer + + # @!attribute reserved_is_ceiling + # Whether `reserved` is an upper bound the batch may finish under. True only for a + # crawl that follows links, whose reachable page count is unknowable until it + # runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact + # page count. + # + # @return [Boolean] + required :reserved_is_ceiling, ContextDev::Internal::Type::Boolean + + # @!attribute submitted + # URLs in the list you sent, before validation and de-duplication. Null for a + # crawl, which is given a source rather than a list. + # + # @return [Integer, nil] + required :submitted, Integer, nil?: true + + # @!method initialize(duplicates:, invalid:, reserved:, reserved_is_ceiling:, submitted:) + # Some parameter documentations has been truncated, see + # {ContextDev::Models::Intake} for more details. + # + # What submission took in, and what it charged for. + # + # @param duplicates [Integer] URLs dropped before reserving because another entry resolved to the same page. N + # + # @param invalid [Integer, nil] URLs from your list rejected as unusable; the same ones are itemised in `invalid + # + # @param reserved [Integer] Pages credits were reserved for. Everything else — progress, the refund, the com + # + # @param reserved_is_ceiling [Boolean] Whether `reserved` is an upper bound the batch may finish under. True only for a + # + # @param submitted [Integer, nil] URLs in the list you sent, before validation and de-duplication. Null for a craw + end + end +end diff --git a/lib/context_dev/models/error_count.rb b/lib/context_dev/models/page_error_count.rb similarity index 90% rename from lib/context_dev/models/error_count.rb rename to lib/context_dev/models/page_error_count.rb index 98470828..d7abd2d9 100644 --- a/lib/context_dev/models/error_count.rb +++ b/lib/context_dev/models/page_error_count.rb @@ -2,7 +2,7 @@ module ContextDev module Models - class ErrorCount < ContextDev::Internal::Type::BaseModel + class PageErrorCount < ContextDev::Internal::Type::BaseModel # @!attribute code # Error code for these failures. # diff --git a/lib/context_dev/models/web_screenshot_params.rb b/lib/context_dev/models/web_screenshot_params.rb index 5cc71b29..ece6dd95 100644 --- a/lib/context_dev/models/web_screenshot_params.rb +++ b/lib/context_dev/models/web_screenshot_params.rb @@ -15,9 +15,8 @@ class WebScreenshotParams < ContextDev::Internal::Type::BaseModel optional :color_scheme, enum: -> { ContextDev::WebScreenshotParams::ColorScheme } # @!attribute country - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). # # @return [Symbol, ContextDev::Models::WebScreenshotParams::Country, nil] optional :country, enum: -> { ContextDev::WebScreenshotParams::Country } @@ -128,7 +127,7 @@ class WebScreenshotParams < ContextDev::Internal::Type::BaseModel # # @param color_scheme [Symbol, ContextDev::Models::WebScreenshotParams::ColorScheme] Optional parameter to choose the site's visual theme in the screenshot. Use 'lig # - # @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https # @@ -168,9 +167,8 @@ module ColorScheme # @return [Array] end - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/lib/context_dev/models/web_web_crawl_md_params.rb b/lib/context_dev/models/web_web_crawl_md_params.rb index 90cc8b61..c2930462 100644 --- a/lib/context_dev/models/web_web_crawl_md_params.rb +++ b/lib/context_dev/models/web_web_crawl_md_params.rb @@ -14,9 +14,8 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel required :url, String # @!attribute country - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). # # @return [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country, nil] optional :country, enum: -> { ContextDev::WebWebCrawlMdParams::Country } @@ -164,7 +163,7 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel # # @param url [String] The starting URL for the crawl (must include http:// or https:// protocol) # - # @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param exclude_selectors [Array] CSS selectors to remove before each crawled page is converted to Markdown. Appli # @@ -206,9 +205,8 @@ class WebWebCrawlMdParams < ContextDev::Internal::Type::BaseModel # # @param request_options [ContextDev::RequestOptions, Hash{Symbol=>Object}] - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/lib/context_dev/models/web_web_scrape_html_params.rb b/lib/context_dev/models/web_web_scrape_html_params.rb index 78b191ff..9c82efde 100644 --- a/lib/context_dev/models/web_web_scrape_html_params.rb +++ b/lib/context_dev/models/web_web_scrape_html_params.rb @@ -24,9 +24,8 @@ class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel nil?: true # @!attribute country - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). # # @return [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country, nil] optional :country, enum: -> { ContextDev::WebWebScrapeHTMLParams::Country } @@ -131,7 +130,7 @@ class WebWebScrapeHTMLParams < ContextDev::Internal::Type::BaseModel # # @param actions [Array, nil] Optional browser actions executed in array order after the page loads and before # - # @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param exclude_selectors [Array, nil] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi # @@ -212,9 +211,8 @@ class Perform < ContextDev::Internal::Type::BaseModel # @return [Array(ContextDev::Models::WebWebScrapeHTMLParams::Action::Wait, ContextDev::Models::WebWebScrapeHTMLParams::Action::Perform)] end - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/lib/context_dev/models/web_web_scrape_md_params.rb b/lib/context_dev/models/web_web_scrape_md_params.rb index 8cc66b78..4ed1739d 100644 --- a/lib/context_dev/models/web_web_scrape_md_params.rb +++ b/lib/context_dev/models/web_web_scrape_md_params.rb @@ -25,9 +25,8 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel nil?: true # @!attribute country - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). # # @return [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country, nil] optional :country, enum: -> { ContextDev::WebWebScrapeMdParams::Country } @@ -150,7 +149,7 @@ class WebWebScrapeMdParams < ContextDev::Internal::Type::BaseModel # # @param actions [Array, nil] Optional browser actions executed in array order after the page loads and before # - # @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param exclude_selectors [Array, nil] CSS selectors to remove before conversion to Markdown. Applied after includeSele # @@ -237,9 +236,8 @@ class Perform < ContextDev::Internal::Type::BaseModel # @return [Array(ContextDev::Models::WebWebScrapeMdParams::Action::Wait, ContextDev::Models::WebWebScrapeMdParams::Action::Perform)] end - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/lib/context_dev/resources/batch.rb b/lib/context_dev/resources/batch.rb index 3b478722..0bd57454 100644 --- a/lib/context_dev/resources/batch.rb +++ b/lib/context_dev/resources/batch.rb @@ -3,9 +3,7 @@ module ContextDev module Resources class Batch - # Check progress and get download links when the batch finishes. Also returns the - # rejected-URL list and webhook signing secret from submission, so nothing is lost - # if the submit response was dropped. + # Check progress, and get download links once the batch finishes. # # @overload retrieve(batch_id, request_options: {}) # @@ -86,9 +84,8 @@ def cancel(batch_id, params = {}) # Some parameter documentations has been truncated, see # {ContextDev::Models::BatchGetResultsParams} for more details. # - # Page through the result records of a finished batch as JSON, in the same order - # as the downloadable result files. Use this instead of downloading and parsing - # the NDJSON files yourself. + # Page through a finished batch's results as JSON instead of downloading the + # NDJSON files. # # @overload get_results(batch_id, cursor: nil, limit: nil, request_options: {}) # diff --git a/lib/context_dev/resources/web.rb b/lib/context_dev/resources/web.rb index a76d7491..0504ffce 100644 --- a/lib/context_dev/resources/web.rb +++ b/lib/context_dev/resources/web.rb @@ -180,7 +180,7 @@ def extract_styleguide(params = {}) # # @param color_scheme [Symbol, ContextDev::Models::WebScreenshotParams::ColorScheme] Optional parameter to choose the site's visual theme in the screenshot. Use 'lig # - # @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebScreenshotParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param direct_url [String] A specific URL to screenshot directly, bypassing domain resolution (e.g., 'https # @@ -285,7 +285,7 @@ def search(params) # # @param url [String] The starting URL for the crawl (must include http:// or https:// protocol) # - # @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebWebCrawlMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param exclude_selectors [Array] CSS selectors to remove before each crawled page is converted to Markdown. Appli # @@ -353,7 +353,7 @@ def web_crawl_md(params) # # @param actions [Array, nil] Optional browser actions executed in array order after the page loads and before # - # @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebWebScrapeHTMLParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param exclude_selectors [Array, nil] CSS selectors to remove from the result. Applied after includeSelectors. Exclusi # @@ -481,7 +481,7 @@ def web_scrape_images(params) # # @param actions [Array, nil] Optional browser actions executed in array order after the page loads and before # - # @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev r + # @param country [Symbol, ContextDev::Models::WebWebScrapeMdParams::Country] Fetch the target page through a residential proxy in this country (ISO 3166-1 al # # @param exclude_selectors [Array, nil] CSS selectors to remove before conversion to Markdown. Applied after includeSele # diff --git a/lib/context_dev/version.rb b/lib/context_dev/version.rb index 0c784f60..6ec90588 100644 --- a/lib/context_dev/version.rb +++ b/lib/context_dev/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ContextDev - VERSION = "2.6.0" + VERSION = "2.7.0" end diff --git a/rbi/context_dev/models.rbi b/rbi/context_dev/models.rbi index 6e887bfa..ab91a86b 100644 --- a/rbi/context_dev/models.rbi +++ b/rbi/context_dev/models.rbi @@ -20,14 +20,16 @@ module ContextDev BrandRetrieveSimplifiedParams = ContextDev::Models::BrandRetrieveSimplifiedParams - Error = ContextDev::Models::Error + CrawlControls = ContextDev::Models::CrawlControls - ErrorCount = ContextDev::Models::ErrorCount + Failure = ContextDev::Models::Failure IndustryRetrieveNaicsParams = ContextDev::Models::IndustryRetrieveNaicsParams IndustryRetrieveSicParams = ContextDev::Models::IndustryRetrieveSicParams + Intake = ContextDev::Models::Intake + MonitorCreateParams = ContextDev::Models::MonitorCreateParams MonitorDeleteParams = ContextDev::Models::MonitorDeleteParams @@ -56,6 +58,8 @@ module ContextDev MonitorUpdateParams = ContextDev::Models::MonitorUpdateParams + PageErrorCount = ContextDev::Models::PageErrorCount + ParseHandleParams = ContextDev::Models::ParseHandleParams UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams diff --git a/rbi/context_dev/models/batch_cancel_response.rbi b/rbi/context_dev/models/batch_cancel_response.rbi index cbbdd28b..e132cdbd 100644 --- a/rbi/context_dev/models/batch_cancel_response.rbi +++ b/rbi/context_dev/models/batch_cancel_response.rbi @@ -11,11 +11,19 @@ module ContextDev ) end - # Batch ID used to retrieve or cancel the job. + # Batch ID. sig { returns(String) } attr_accessor :id - # Reserved and used credits. + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + sig { returns(T.nilable(ContextDev::CrawlControls)) } + attr_reader :crawl + + sig { params(crawl: T.nilable(ContextDev::CrawlControls::OrHash)).void } + attr_writer :crawl + + # What this batch cost so far. sig { returns(ContextDev::Models::BatchCancelResponse::Credits) } attr_reader :credits @@ -26,35 +34,30 @@ module ContextDev end attr_writer :credits - # Why the batch failed. - sig { returns(T.nilable(ContextDev::Error)) } - attr_reader :error - - sig { params(error: T.nilable(ContextDev::Error::OrHash)).void } - attr_writer :error - - # Page failures grouped by error code. - sig { returns(T::Array[ContextDev::ErrorCount]) } - attr_accessor :errors + # What each page is returned as. + sig do + returns(ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol) + end + attr_accessor :format_ - # Submission counts. - sig { returns(ContextDev::Models::BatchCancelResponse::Input) } + # What submission took in, and what it charged for. + sig { returns(ContextDev::Intake) } attr_reader :input - sig do - params( - input: ContextDev::Models::BatchCancelResponse::Input::OrHash - ).void - end + sig { params(input: ContextDev::Intake::OrHash).void } attr_writer :input - # How pages are selected. + # How pages were selected. sig do returns(ContextDev::Models::BatchCancelResponse::Mode::TaggedSymbol) end attr_accessor :mode - # Current processing counts. Use `status` to check completion. + # Page failures so far, grouped by error code and sorted by count. + sig { returns(T::Array[ContextDev::PageErrorCount]) } + attr_accessor :page_errors + + # How far the batch got before cancellation. sig { returns(ContextDev::Models::BatchCancelResponse::Progress) } attr_reader :progress @@ -65,22 +68,8 @@ module ContextDev end attr_writer :progress - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. - sig do - returns(T.nilable(ContextDev::Models::BatchCancelResponse::Results)) - end - attr_reader :results - - sig do - params( - results: - T.nilable(ContextDev::Models::BatchCancelResponse::Results::OrHash) - ).void - end - attr_writer :results - - # Current state. `completed`, `cancelled`, and `failed` are final. + # Always `cancelling`. Work already in flight finishes; the batch reaches + # `cancelled` shortly after. sig do returns(ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol) end @@ -90,6 +79,7 @@ module ContextDev sig { returns(T::Array[String]) } attr_accessor :tags + # There is no finish time yet — the batch is still winding down. sig { returns(ContextDev::Models::BatchCancelResponse::Timing) } attr_reader :timing @@ -100,12 +90,6 @@ module ContextDev end attr_writer :timing - # Output format. - sig do - returns(ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol) - end - attr_accessor :type - # API key usage for this request. sig do returns(T.nilable(ContextDev::Models::BatchCancelResponse::KeyMetadata)) @@ -123,47 +107,45 @@ module ContextDev sig do params( id: String, + crawl: T.nilable(ContextDev::CrawlControls::OrHash), credits: ContextDev::Models::BatchCancelResponse::Credits::OrHash, - error: T.nilable(ContextDev::Error::OrHash), - errors: T::Array[ContextDev::ErrorCount::OrHash], - input: ContextDev::Models::BatchCancelResponse::Input::OrHash, + format_: ContextDev::Models::BatchCancelResponse::Format::OrSymbol, + input: ContextDev::Intake::OrHash, mode: ContextDev::Models::BatchCancelResponse::Mode::OrSymbol, + page_errors: T::Array[ContextDev::PageErrorCount::OrHash], progress: ContextDev::Models::BatchCancelResponse::Progress::OrHash, - results: - T.nilable(ContextDev::Models::BatchCancelResponse::Results::OrHash), status: ContextDev::Models::BatchCancelResponse::Status::OrSymbol, tags: T::Array[String], timing: ContextDev::Models::BatchCancelResponse::Timing::OrHash, - type: ContextDev::Models::BatchCancelResponse::Type::OrSymbol, key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata::OrHash ).returns(T.attached_class) end def self.new( - # Batch ID used to retrieve or cancel the job. + # Batch ID. id:, - # Reserved and used credits. + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + crawl:, + # What this batch cost so far. credits:, - # Why the batch failed. - error:, - # Page failures grouped by error code. - errors:, - # Submission counts. + # What each page is returned as. + format_:, + # What submission took in, and what it charged for. input:, - # How pages are selected. + # How pages were selected. mode:, - # Current processing counts. Use `status` to check completion. + # Page failures so far, grouped by error code and sorted by count. + page_errors:, + # How far the batch got before cancellation. progress:, - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. - results:, - # Current state. `completed`, `cancelled`, and `failed` are final. + # Always `cancelling`. Work already in flight finishes; the batch reaches + # `cancelled` shortly after. status:, # Tags stored on the batch at submission. tags:, + # There is no finish time yet — the batch is still winding down. timing:, - # Output format. - type:, # API key usage for this request. key_metadata: nil ) @@ -173,19 +155,18 @@ module ContextDev override.returns( { id: String, + crawl: T.nilable(ContextDev::CrawlControls), credits: ContextDev::Models::BatchCancelResponse::Credits, - error: T.nilable(ContextDev::Error), - errors: T::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchCancelResponse::Input, + format_: + ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol, + input: ContextDev::Intake, mode: ContextDev::Models::BatchCancelResponse::Mode::TaggedSymbol, + page_errors: T::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchCancelResponse::Progress, - results: - T.nilable(ContextDev::Models::BatchCancelResponse::Results), status: ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol, tags: T::Array[String], timing: ContextDev::Models::BatchCancelResponse::Timing, - type: ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol, key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata } ) @@ -202,92 +183,58 @@ module ContextDev ) end - # Credits used by successful pages. + # Credits debited at submission. The unspent remainder is refunded once the batch + # settles — read `credits.refunded` from GET /batch/{batch_id} then. sig { returns(Integer) } - attr_accessor :charged + attr_accessor :reserved - # Credits reserved when the batch was accepted. - sig { returns(Integer) } - attr_accessor :estimated - - # Reserved and used credits. - sig do - params(charged: Integer, estimated: Integer).returns(T.attached_class) - end + # What this batch cost so far. + sig { params(reserved: Integer).returns(T.attached_class) } def self.new( - # Credits used by successful pages. - charged:, - # Credits reserved when the batch was accepted. - estimated: + # Credits debited at submission. The unspent remainder is refunded once the batch + # settles — read `credits.refunded` from GET /batch/{batch_id} then. + reserved: ) end - sig { override.returns({ charged: Integer, estimated: Integer }) } + sig { override.returns({ reserved: Integer }) } def to_hash end end - class Input < ContextDev::Internal::Type::BaseModel - OrHash = + # What each page is returned as. + module Format + extend ContextDev::Internal::Type::Enum + + TaggedSymbol = T.type_alias do - T.any( - ContextDev::Models::BatchCancelResponse::Input, - ContextDev::Internal::AnyHash - ) + T.all(Symbol, ContextDev::Models::BatchCancelResponse::Format) end + OrSymbol = T.type_alias { T.any(Symbol, String) } - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - sig { returns(Integer) } - attr_accessor :accepted - - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - sig { returns(Integer) } - attr_accessor :duplicates - - # Pages rejected during validation. - sig { returns(Integer) } - attr_accessor :invalid - - # Pages submitted before validation. For a crawl, the page limit. - sig { returns(Integer) } - attr_accessor :submitted - - # Submission counts. - sig do - params( - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - ).returns(T.attached_class) - end - def self.new( - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - accepted:, - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - duplicates:, - # Pages rejected during validation. - invalid:, - # Pages submitted before validation. For a crawl, the page limit. - submitted: - ) - end + MARKDOWN = + T.let( + :markdown, + ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol + ) + HTML = + T.let( + :html, + ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol + ) sig do override.returns( - { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } + T::Array[ + ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol + ] ) end - def to_hash + def self.values end end - # How pages are selected. + # How pages were selected. module Mode extend ContextDev::Internal::Type::Enum @@ -328,32 +275,30 @@ module ContextDev ) end - # Pages that could not be scraped. + # Pages that could not be scraped before the request landed. sig { returns(Integer) } attr_accessor :failed - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages that will now be skipped, and refunded when the batch settles. sig { returns(Integer) } attr_accessor :pending - # Pages scraped successfully. + # Pages scraped successfully before the request landed. sig { returns(Integer) } attr_accessor :succeeded - # Current processing counts. Use `status` to check completion. + # How far the batch got before cancellation. sig do params(failed: Integer, pending: Integer, succeeded: Integer).returns( T.attached_class ) end def self.new( - # Pages that could not be scraped. + # Pages that could not be scraped before the request landed. failed:, - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages that will now be skipped, and refunded when the batch settles. pending:, - # Pages scraped successfully. + # Pages scraped successfully before the request landed. succeeded: ) end @@ -367,103 +312,8 @@ module ContextDev end end - class Results < ContextDev::Internal::Type::BaseModel - OrHash = - T.type_alias do - T.any( - ContextDev::Models::BatchCancelResponse::Results, - ContextDev::Internal::AnyHash - ) - end - - # When the download URLs expire. - sig { returns(String) } - attr_accessor :expires_at - - # Result files. Order is not guaranteed. - sig do - returns( - T::Array[ContextDev::Models::BatchCancelResponse::Results::File] - ) - end - attr_accessor :files - - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. - sig do - params( - expires_at: String, - files: - T::Array[ - ContextDev::Models::BatchCancelResponse::Results::File::OrHash - ] - ).returns(T.attached_class) - end - def self.new( - # When the download URLs expire. - expires_at:, - # Result files. Order is not guaranteed. - files: - ) - end - - sig do - override.returns( - { - expires_at: String, - files: - T::Array[ContextDev::Models::BatchCancelResponse::Results::File] - } - ) - end - def to_hash - end - - class File < ContextDev::Internal::Type::BaseModel - OrHash = - T.type_alias do - T.any( - ContextDev::Models::BatchCancelResponse::Results::File, - ContextDev::Internal::AnyHash - ) - end - - # Compressed file size in bytes. - sig { returns(Integer) } - attr_accessor :bytes - - # Results in this file. - sig { returns(Integer) } - attr_accessor :items - - # Temporary URL for a gzipped NDJSON file. - sig { returns(String) } - attr_accessor :url - - sig do - params(bytes: Integer, items: Integer, url: String).returns( - T.attached_class - ) - end - def self.new( - # Compressed file size in bytes. - bytes:, - # Results in this file. - items:, - # Temporary URL for a gzipped NDJSON file. - url: - ) - end - - sig do - override.returns({ bytes: Integer, items: Integer, url: String }) - end - def to_hash - end - end - end - - # Current state. `completed`, `cancelled`, and `failed` are final. + # Always `cancelling`. Work already in flight finishes; the batch reaches + # `cancelled` shortly after. module Status extend ContextDev::Internal::Type::Enum @@ -473,36 +323,11 @@ module ContextDev end OrSymbol = T.type_alias { T.any(Symbol, String) } - QUEUED = - T.let( - :queued, - ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol - ) - RUNNING = - T.let( - :running, - ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol - ) CANCELLING = T.let( :cancelling, ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol ) - COMPLETED = - T.let( - :completed, - ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol - ) - CANCELLED = - T.let( - :cancelled, - ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol - ) - FAILED = - T.let( - :failed, - ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol - ) sig do override.returns( @@ -524,80 +349,37 @@ module ContextDev ) end - # When processing finished. Null while active. - sig { returns(T.nilable(String)) } - attr_accessor :completed_at - # When the batch was created. sig { returns(String) } attr_accessor :created_at - # When processing started. Null while queued. + # When processing started. Null if it was cancelled while still queued. sig { returns(T.nilable(String)) } attr_accessor :started_at + # There is no finish time yet — the batch is still winding down. sig do - params( - completed_at: T.nilable(String), - created_at: String, - started_at: T.nilable(String) - ).returns(T.attached_class) + params(created_at: String, started_at: T.nilable(String)).returns( + T.attached_class + ) end def self.new( - # When processing finished. Null while active. - completed_at:, # When the batch was created. created_at:, - # When processing started. Null while queued. + # When processing started. Null if it was cancelled while still queued. started_at: ) end sig do override.returns( - { - completed_at: T.nilable(String), - created_at: String, - started_at: T.nilable(String) - } + { created_at: String, started_at: T.nilable(String) } ) end def to_hash end end - # Output format. - module Type - extend ContextDev::Internal::Type::Enum - - TaggedSymbol = - T.type_alias do - T.all(Symbol, ContextDev::Models::BatchCancelResponse::Type) - end - OrSymbol = T.type_alias { T.any(Symbol, String) } - - MARKDOWN = - T.let( - :markdown, - ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol - ) - HTML = - T.let( - :html, - ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol - ) - - sig do - override.returns( - T::Array[ - ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol - ] - ) - end - def self.values - end - end - class KeyMetadata < ContextDev::Internal::Type::BaseModel OrHash = T.type_alias do diff --git a/rbi/context_dev/models/batch_list_response.rbi b/rbi/context_dev/models/batch_list_response.rbi index 0aa18e00..23c402f2 100644 --- a/rbi/context_dev/models/batch_list_response.rbi +++ b/rbi/context_dev/models/batch_list_response.rbi @@ -100,7 +100,15 @@ module ContextDev sig { returns(String) } attr_accessor :id - # Reserved and used credits. + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + sig { returns(T.nilable(ContextDev::CrawlControls)) } + attr_reader :crawl + + sig { params(crawl: T.nilable(ContextDev::CrawlControls::OrHash)).void } + attr_writer :crawl + + # What this batch has done to your credit balance. sig { returns(ContextDev::Models::BatchListResponse::Data::Credits) } attr_reader :credits @@ -112,29 +120,31 @@ module ContextDev end attr_writer :credits - # Why the batch failed. - sig { returns(T.nilable(ContextDev::Error)) } - attr_reader :error + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. + sig { returns(T.nilable(ContextDev::Failure)) } + attr_reader :failure - sig { params(error: T.nilable(ContextDev::Error::OrHash)).void } - attr_writer :error + sig { params(failure: T.nilable(ContextDev::Failure::OrHash)).void } + attr_writer :failure - # Page failures grouped by error code. - sig { returns(T::Array[ContextDev::ErrorCount]) } - attr_accessor :errors + # What each page is returned as. Matches `input.data.format` on the submit + # request. + sig do + returns( + ContextDev::Models::BatchListResponse::Data::Format::TaggedSymbol + ) + end + attr_accessor :format_ - # Submission counts. - sig { returns(ContextDev::Models::BatchListResponse::Data::Input) } + # What submission took in, and what it charged for. + sig { returns(ContextDev::Intake) } attr_reader :input - sig do - params( - input: ContextDev::Models::BatchListResponse::Data::Input::OrHash - ).void - end + sig { params(input: ContextDev::Intake::OrHash).void } attr_writer :input - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. sig do returns( ContextDev::Models::BatchListResponse::Data::Mode::TaggedSymbol @@ -142,7 +152,12 @@ module ContextDev end attr_accessor :mode - # Current processing counts. Use `status` to check completion. + # Individual page failures grouped by error code, sorted by count. Unrelated to + # `failure`, which is the batch itself failing. + sig { returns(T::Array[ContextDev::PageErrorCount]) } + attr_accessor :page_errors + + # Pages attempted so far. Use `status` to check completion. sig { returns(ContextDev::Models::BatchListResponse::Data::Progress) } attr_reader :progress @@ -154,8 +169,8 @@ module ContextDev end attr_writer :progress - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. sig do returns( T.nilable(ContextDev::Models::BatchListResponse::Data::Results) @@ -195,24 +210,19 @@ module ContextDev end attr_writer :timing - # Output format. - sig do - returns( - ContextDev::Models::BatchListResponse::Data::Type::TaggedSymbol - ) - end - attr_accessor :type - # An asynchronous web scraping job. sig do params( id: String, + crawl: T.nilable(ContextDev::CrawlControls::OrHash), credits: ContextDev::Models::BatchListResponse::Data::Credits::OrHash, - error: T.nilable(ContextDev::Error::OrHash), - errors: T::Array[ContextDev::ErrorCount::OrHash], - input: ContextDev::Models::BatchListResponse::Data::Input::OrHash, + failure: T.nilable(ContextDev::Failure::OrHash), + format_: + ContextDev::Models::BatchListResponse::Data::Format::OrSymbol, + input: ContextDev::Intake::OrHash, mode: ContextDev::Models::BatchListResponse::Data::Mode::OrSymbol, + page_errors: T::Array[ContextDev::PageErrorCount::OrHash], progress: ContextDev::Models::BatchListResponse::Data::Progress::OrHash, results: @@ -222,35 +232,40 @@ module ContextDev status: ContextDev::Models::BatchListResponse::Data::Status::OrSymbol, tags: T::Array[String], - timing: ContextDev::Models::BatchListResponse::Data::Timing::OrHash, - type: ContextDev::Models::BatchListResponse::Data::Type::OrSymbol + timing: ContextDev::Models::BatchListResponse::Data::Timing::OrHash ).returns(T.attached_class) end def self.new( # Batch ID used to retrieve or cancel the job. id:, - # Reserved and used credits. + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + crawl:, + # What this batch has done to your credit balance. credits:, - # Why the batch failed. - error:, - # Page failures grouped by error code. - errors:, - # Submission counts. + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. + failure:, + # What each page is returned as. Matches `input.data.format` on the submit + # request. + format_:, + # What submission took in, and what it charged for. input:, - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. mode:, - # Current processing counts. Use `status` to check completion. + # Individual page failures grouped by error code, sorted by count. Unrelated to + # `failure`, which is the batch itself failing. + page_errors:, + # Pages attempted so far. Use `status` to check completion. progress:, - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. results:, # Current state. `completed`, `cancelled`, and `failed` are final. status:, # Tags stored on the batch at submission. tags:, - timing:, - # Output format. - type: + timing: ) end @@ -258,21 +273,22 @@ module ContextDev override.returns( { id: String, + crawl: T.nilable(ContextDev::CrawlControls), credits: ContextDev::Models::BatchListResponse::Data::Credits, - error: T.nilable(ContextDev::Error), - errors: T::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchListResponse::Data::Input, + failure: T.nilable(ContextDev::Failure), + format_: + ContextDev::Models::BatchListResponse::Data::Format::TaggedSymbol, + input: ContextDev::Intake, mode: ContextDev::Models::BatchListResponse::Data::Mode::TaggedSymbol, + page_errors: T::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchListResponse::Data::Progress, results: T.nilable(ContextDev::Models::BatchListResponse::Data::Results), status: ContextDev::Models::BatchListResponse::Data::Status::TaggedSymbol, tags: T::Array[String], - timing: ContextDev::Models::BatchListResponse::Data::Timing, - type: - ContextDev::Models::BatchListResponse::Data::Type::TaggedSymbol + timing: ContextDev::Models::BatchListResponse::Data::Timing } ) end @@ -288,94 +304,83 @@ module ContextDev ) end - # Credits used by successful pages. + # `reserved` minus `refunded` — what the batch has cost so far. Equal to + # `reserved` until the batch settles. sig { returns(Integer) } - attr_accessor :charged + attr_accessor :net - # Credits reserved when the batch was accepted. + # Credits returned for pages that did not succeed. Stays 0 until the batch reaches + # a final status, then settles in one movement. sig { returns(Integer) } - attr_accessor :estimated + attr_accessor :refunded - # Reserved and used credits. + # Credits debited from your balance the moment the batch was accepted. This is a + # charge, not a forecast — the whole amount leaves the balance up front. + sig { returns(Integer) } + attr_accessor :reserved + + # What this batch has done to your credit balance. sig do - params(charged: Integer, estimated: Integer).returns( + params(net: Integer, refunded: Integer, reserved: Integer).returns( T.attached_class ) end def self.new( - # Credits used by successful pages. - charged:, - # Credits reserved when the batch was accepted. - estimated: + # `reserved` minus `refunded` — what the batch has cost so far. Equal to + # `reserved` until the batch settles. + net:, + # Credits returned for pages that did not succeed. Stays 0 until the batch reaches + # a final status, then settles in one movement. + refunded:, + # Credits debited from your balance the moment the batch was accepted. This is a + # charge, not a forecast — the whole amount leaves the balance up front. + reserved: ) end - sig { override.returns({ charged: Integer, estimated: Integer }) } + sig do + override.returns( + { net: Integer, refunded: Integer, reserved: Integer } + ) + end def to_hash end end - class Input < ContextDev::Internal::Type::BaseModel - OrHash = + # What each page is returned as. Matches `input.data.format` on the submit + # request. + module Format + extend ContextDev::Internal::Type::Enum + + TaggedSymbol = T.type_alias do - T.any( - ContextDev::Models::BatchListResponse::Data::Input, - ContextDev::Internal::AnyHash - ) + T.all(Symbol, ContextDev::Models::BatchListResponse::Data::Format) end + OrSymbol = T.type_alias { T.any(Symbol, String) } - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - sig { returns(Integer) } - attr_accessor :accepted - - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - sig { returns(Integer) } - attr_accessor :duplicates - - # Pages rejected during validation. - sig { returns(Integer) } - attr_accessor :invalid - - # Pages submitted before validation. For a crawl, the page limit. - sig { returns(Integer) } - attr_accessor :submitted - - # Submission counts. - sig do - params( - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - ).returns(T.attached_class) - end - def self.new( - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - accepted:, - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - duplicates:, - # Pages rejected during validation. - invalid:, - # Pages submitted before validation. For a crawl, the page limit. - submitted: - ) - end + MARKDOWN = + T.let( + :markdown, + ContextDev::Models::BatchListResponse::Data::Format::TaggedSymbol + ) + HTML = + T.let( + :html, + ContextDev::Models::BatchListResponse::Data::Format::TaggedSymbol + ) sig do override.returns( - { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } + T::Array[ + ContextDev::Models::BatchListResponse::Data::Format::TaggedSymbol + ] ) end - def to_hash + def self.values end end - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. module Mode extend ContextDev::Internal::Type::Enum @@ -420,8 +425,9 @@ module ContextDev sig { returns(Integer) } attr_accessor :failed - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + # never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + # final, because its unspent budget was never real pages. sig { returns(Integer) } attr_accessor :pending @@ -429,7 +435,7 @@ module ContextDev sig { returns(Integer) } attr_accessor :succeeded - # Current processing counts. Use `status` to check completion. + # Pages attempted so far. Use `status` to check completion. sig do params( failed: Integer, @@ -440,8 +446,9 @@ module ContextDev def self.new( # Pages that could not be scraped. failed:, - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + # never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + # final, because its unspent budget was never real pages. pending:, # Pages scraped successfully. succeeded: @@ -480,8 +487,8 @@ module ContextDev end attr_accessor :files - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. sig do params( expires_at: String, @@ -659,38 +666,6 @@ module ContextDev def to_hash end end - - # Output format. - module Type - extend ContextDev::Internal::Type::Enum - - TaggedSymbol = - T.type_alias do - T.all(Symbol, ContextDev::Models::BatchListResponse::Data::Type) - end - OrSymbol = T.type_alias { T.any(Symbol, String) } - - MARKDOWN = - T.let( - :markdown, - ContextDev::Models::BatchListResponse::Data::Type::TaggedSymbol - ) - HTML = - T.let( - :html, - ContextDev::Models::BatchListResponse::Data::Type::TaggedSymbol - ) - - sig do - override.returns( - T::Array[ - ContextDev::Models::BatchListResponse::Data::Type::TaggedSymbol - ] - ) - end - def self.values - end - end end class KeyMetadata < ContextDev::Internal::Type::BaseModel diff --git a/rbi/context_dev/models/batch_retrieve_response.rbi b/rbi/context_dev/models/batch_retrieve_response.rbi index a86b6453..1acf02e8 100644 --- a/rbi/context_dev/models/batch_retrieve_response.rbi +++ b/rbi/context_dev/models/batch_retrieve_response.rbi @@ -15,7 +15,15 @@ module ContextDev sig { returns(String) } attr_accessor :id - # Reserved and used credits. + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + sig { returns(T.nilable(ContextDev::CrawlControls)) } + attr_reader :crawl + + sig { params(crawl: T.nilable(ContextDev::CrawlControls::OrHash)).void } + attr_writer :crawl + + # What this batch has done to your credit balance. sig { returns(ContextDev::Models::BatchRetrieveResponse::Credits) } attr_reader :credits @@ -26,26 +34,26 @@ module ContextDev end attr_writer :credits - # Why the batch failed. - sig { returns(T.nilable(ContextDev::Error)) } - attr_reader :error + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. + sig { returns(T.nilable(ContextDev::Failure)) } + attr_reader :failure - sig { params(error: T.nilable(ContextDev::Error::OrHash)).void } - attr_writer :error + sig { params(failure: T.nilable(ContextDev::Failure::OrHash)).void } + attr_writer :failure - # Page failures grouped by error code. - sig { returns(T::Array[ContextDev::ErrorCount]) } - attr_accessor :errors + # What each page is returned as. Matches `input.data.format` on the submit + # request. + sig do + returns(ContextDev::Models::BatchRetrieveResponse::Format::TaggedSymbol) + end + attr_accessor :format_ - # Submission counts. - sig { returns(ContextDev::Models::BatchRetrieveResponse::Input) } + # What submission took in, and what it charged for. + sig { returns(ContextDev::Intake) } attr_reader :input - sig do - params( - input: ContextDev::Models::BatchRetrieveResponse::Input::OrHash - ).void - end + sig { params(input: ContextDev::Intake::OrHash).void } attr_writer :input # Rejected URLs, up to 100. These are not charged. @@ -54,13 +62,18 @@ module ContextDev end attr_accessor :invalid_urls - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. sig do returns(ContextDev::Models::BatchRetrieveResponse::Mode::TaggedSymbol) end attr_accessor :mode - # Current processing counts. Use `status` to check completion. + # Individual page failures grouped by error code, sorted by count. Unrelated to + # `failure`, which is the batch itself failing. + sig { returns(T::Array[ContextDev::PageErrorCount]) } + attr_accessor :page_errors + + # Pages attempted so far. Use `status` to check completion. sig { returns(ContextDev::Models::BatchRetrieveResponse::Progress) } attr_reader :progress @@ -71,8 +84,8 @@ module ContextDev end attr_writer :progress - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. sig do returns(T.nilable(ContextDev::Models::BatchRetrieveResponse::Results)) end @@ -108,12 +121,6 @@ module ContextDev end attr_writer :timing - # Output format. - sig do - returns(ContextDev::Models::BatchRetrieveResponse::Type::TaggedSymbol) - end - attr_accessor :type - # API key usage for this request. sig do returns( @@ -130,25 +137,20 @@ module ContextDev end attr_writer :key_metadata - # Webhook signing secret. Also returned by GET /batch/{batch_id}. - sig { returns(T.nilable(String)) } - attr_reader :webhook_secret - - sig { params(webhook_secret: String).void } - attr_writer :webhook_secret - sig do params( id: String, + crawl: T.nilable(ContextDev::CrawlControls::OrHash), credits: ContextDev::Models::BatchRetrieveResponse::Credits::OrHash, - error: T.nilable(ContextDev::Error::OrHash), - errors: T::Array[ContextDev::ErrorCount::OrHash], - input: ContextDev::Models::BatchRetrieveResponse::Input::OrHash, + failure: T.nilable(ContextDev::Failure::OrHash), + format_: ContextDev::Models::BatchRetrieveResponse::Format::OrSymbol, + input: ContextDev::Intake::OrHash, invalid_urls: T::Array[ ContextDev::Models::BatchRetrieveResponse::InvalidURL::OrHash ], mode: ContextDev::Models::BatchRetrieveResponse::Mode::OrSymbol, + page_errors: T::Array[ContextDev::PageErrorCount::OrHash], progress: ContextDev::Models::BatchRetrieveResponse::Progress::OrHash, results: T.nilable( @@ -157,43 +159,45 @@ module ContextDev status: ContextDev::Models::BatchRetrieveResponse::Status::OrSymbol, tags: T::Array[String], timing: ContextDev::Models::BatchRetrieveResponse::Timing::OrHash, - type: ContextDev::Models::BatchRetrieveResponse::Type::OrSymbol, key_metadata: - ContextDev::Models::BatchRetrieveResponse::KeyMetadata::OrHash, - webhook_secret: String + ContextDev::Models::BatchRetrieveResponse::KeyMetadata::OrHash ).returns(T.attached_class) end def self.new( # Batch ID used to retrieve or cancel the job. id:, - # Reserved and used credits. + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + crawl:, + # What this batch has done to your credit balance. credits:, - # Why the batch failed. - error:, - # Page failures grouped by error code. - errors:, - # Submission counts. + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. + failure:, + # What each page is returned as. Matches `input.data.format` on the submit + # request. + format_:, + # What submission took in, and what it charged for. input:, # Rejected URLs, up to 100. These are not charged. invalid_urls:, - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. mode:, - # Current processing counts. Use `status` to check completion. + # Individual page failures grouped by error code, sorted by count. Unrelated to + # `failure`, which is the batch itself failing. + page_errors:, + # Pages attempted so far. Use `status` to check completion. progress:, - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. results:, # Current state. `completed`, `cancelled`, and `failed` are final. status:, # Tags stored on the batch at submission. tags:, timing:, - # Output format. - type:, # API key usage for this request. - key_metadata: nil, - # Webhook signing secret. Also returned by GET /batch/{batch_id}. - webhook_secret: nil + key_metadata: nil ) end @@ -201,13 +205,16 @@ module ContextDev override.returns( { id: String, + crawl: T.nilable(ContextDev::CrawlControls), credits: ContextDev::Models::BatchRetrieveResponse::Credits, - error: T.nilable(ContextDev::Error), - errors: T::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchRetrieveResponse::Input, + failure: T.nilable(ContextDev::Failure), + format_: + ContextDev::Models::BatchRetrieveResponse::Format::TaggedSymbol, + input: ContextDev::Intake, invalid_urls: T::Array[ContextDev::Models::BatchRetrieveResponse::InvalidURL], mode: ContextDev::Models::BatchRetrieveResponse::Mode::TaggedSymbol, + page_errors: T::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchRetrieveResponse::Progress, results: T.nilable(ContextDev::Models::BatchRetrieveResponse::Results), @@ -215,10 +222,7 @@ module ContextDev ContextDev::Models::BatchRetrieveResponse::Status::TaggedSymbol, tags: T::Array[String], timing: ContextDev::Models::BatchRetrieveResponse::Timing, - type: ContextDev::Models::BatchRetrieveResponse::Type::TaggedSymbol, - key_metadata: - ContextDev::Models::BatchRetrieveResponse::KeyMetadata, - webhook_secret: String + key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata } ) end @@ -234,88 +238,79 @@ module ContextDev ) end - # Credits used by successful pages. + # `reserved` minus `refunded` — what the batch has cost so far. Equal to + # `reserved` until the batch settles. sig { returns(Integer) } - attr_accessor :charged + attr_accessor :net - # Credits reserved when the batch was accepted. + # Credits returned for pages that did not succeed. Stays 0 until the batch reaches + # a final status, then settles in one movement. sig { returns(Integer) } - attr_accessor :estimated + attr_accessor :refunded - # Reserved and used credits. + # Credits debited from your balance the moment the batch was accepted. This is a + # charge, not a forecast — the whole amount leaves the balance up front. + sig { returns(Integer) } + attr_accessor :reserved + + # What this batch has done to your credit balance. sig do - params(charged: Integer, estimated: Integer).returns(T.attached_class) + params(net: Integer, refunded: Integer, reserved: Integer).returns( + T.attached_class + ) end def self.new( - # Credits used by successful pages. - charged:, - # Credits reserved when the batch was accepted. - estimated: + # `reserved` minus `refunded` — what the batch has cost so far. Equal to + # `reserved` until the batch settles. + net:, + # Credits returned for pages that did not succeed. Stays 0 until the batch reaches + # a final status, then settles in one movement. + refunded:, + # Credits debited from your balance the moment the batch was accepted. This is a + # charge, not a forecast — the whole amount leaves the balance up front. + reserved: ) end - sig { override.returns({ charged: Integer, estimated: Integer }) } + sig do + override.returns( + { net: Integer, refunded: Integer, reserved: Integer } + ) + end def to_hash end end - class Input < ContextDev::Internal::Type::BaseModel - OrHash = + # What each page is returned as. Matches `input.data.format` on the submit + # request. + module Format + extend ContextDev::Internal::Type::Enum + + TaggedSymbol = T.type_alias do - T.any( - ContextDev::Models::BatchRetrieveResponse::Input, - ContextDev::Internal::AnyHash - ) + T.all(Symbol, ContextDev::Models::BatchRetrieveResponse::Format) end + OrSymbol = T.type_alias { T.any(Symbol, String) } - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - sig { returns(Integer) } - attr_accessor :accepted - - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - sig { returns(Integer) } - attr_accessor :duplicates - - # Pages rejected during validation. - sig { returns(Integer) } - attr_accessor :invalid - - # Pages submitted before validation. For a crawl, the page limit. - sig { returns(Integer) } - attr_accessor :submitted - - # Submission counts. - sig do - params( - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - ).returns(T.attached_class) - end - def self.new( - # Pages accepted, or the crawl page limit. Credits are reserved for this count. - accepted:, - # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls. - duplicates:, - # Pages rejected during validation. - invalid:, - # Pages submitted before validation. For a crawl, the page limit. - submitted: - ) - end + MARKDOWN = + T.let( + :markdown, + ContextDev::Models::BatchRetrieveResponse::Format::TaggedSymbol + ) + HTML = + T.let( + :html, + ContextDev::Models::BatchRetrieveResponse::Format::TaggedSymbol + ) sig do override.returns( - { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } + T::Array[ + ContextDev::Models::BatchRetrieveResponse::Format::TaggedSymbol + ] ) end - def to_hash + def self.values end end @@ -350,7 +345,7 @@ module ContextDev end end - # How pages are selected. + # How pages were selected. Matches `input.mode` on the submit request. module Mode extend ContextDev::Internal::Type::Enum @@ -395,8 +390,9 @@ module ContextDev sig { returns(Integer) } attr_accessor :failed - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + # never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + # final, because its unspent budget was never real pages. sig { returns(Integer) } attr_accessor :pending @@ -404,7 +400,7 @@ module ContextDev sig { returns(Integer) } attr_accessor :succeeded - # Current processing counts. Use `status` to check completion. + # Pages attempted so far. Use `status` to check completion. sig do params(failed: Integer, pending: Integer, succeeded: Integer).returns( T.attached_class @@ -413,8 +409,9 @@ module ContextDev def self.new( # Pages that could not be scraped. failed:, - # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can - # finish under its page limit when the site has no more reachable pages. + # Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it + # never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once + # final, because its unspent budget was never real pages. pending:, # Pages scraped successfully. succeeded: @@ -451,8 +448,8 @@ module ContextDev end attr_accessor :files - # Download links available when the batch finishes. GET /batch/{batch_id}/results - # serves the same records as paginated JSON. + # Download links, available once the batch reaches a final status and null before + # then. GET /batch/{batch_id}/results serves the same records as paginated JSON. sig do params( expires_at: String, @@ -631,38 +628,6 @@ module ContextDev end end - # Output format. - module Type - extend ContextDev::Internal::Type::Enum - - TaggedSymbol = - T.type_alias do - T.all(Symbol, ContextDev::Models::BatchRetrieveResponse::Type) - end - OrSymbol = T.type_alias { T.any(Symbol, String) } - - MARKDOWN = - T.let( - :markdown, - ContextDev::Models::BatchRetrieveResponse::Type::TaggedSymbol - ) - HTML = - T.let( - :html, - ContextDev::Models::BatchRetrieveResponse::Type::TaggedSymbol - ) - - sig do - override.returns( - T::Array[ - ContextDev::Models::BatchRetrieveResponse::Type::TaggedSymbol - ] - ) - end - def self.values - end - end - class KeyMetadata < ContextDev::Internal::Type::BaseModel OrHash = T.type_alias do diff --git a/rbi/context_dev/models/crawl_controls.rbi b/rbi/context_dev/models/crawl_controls.rbi new file mode 100644 index 00000000..ccf7fd34 --- /dev/null +++ b/rbi/context_dev/models/crawl_controls.rbi @@ -0,0 +1,252 @@ +# typed: strong + +module ContextDev + module Models + class CrawlControls < ContextDev::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any(ContextDev::CrawlControls, ContextDev::Internal::AnyHash) + end + + # Whether links to subdomains were followed. Always false for a sitemap crawl. + sig { returns(T::Boolean) } + attr_accessor :follow_subdomains + + # Link depth limit. Always 0 for a sitemap crawl, which never follows links off + # its URLs; null when a `start_url` crawl set no limit. + sig { returns(T.nilable(Integer)) } + attr_accessor :max_depth + + # The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs + # its sitemap actually lists, up to this many, so `input.reserved` is often lower. + sig { returns(Integer) } + attr_accessor :max_pages + + # Where the crawl started. + sig { returns(ContextDev::CrawlControls::Source::Variants) } + attr_accessor :source + + # RE2 pattern URLs had to match to be crawled. Null when the crawl set none. + sig { returns(T.nilable(String)) } + attr_accessor :url_pattern + + # The crawl controls as submitted, so the limits requested can be compared against + # what the crawl reached. + sig do + params( + follow_subdomains: T::Boolean, + max_depth: T.nilable(Integer), + max_pages: Integer, + source: + T.any( + ContextDev::CrawlControls::Source::UnionMember0::OrHash, + ContextDev::CrawlControls::Source::UnionMember1::OrHash + ), + url_pattern: T.nilable(String) + ).returns(T.attached_class) + end + def self.new( + # Whether links to subdomains were followed. Always false for a sitemap crawl. + follow_subdomains:, + # Link depth limit. Always 0 for a sitemap crawl, which never follows links off + # its URLs; null when a `start_url` crawl set no limit. + max_depth:, + # The `maxUrls` submitted with the crawl. A sitemap crawl scrapes only the URLs + # its sitemap actually lists, up to this many, so `input.reserved` is often lower. + max_pages:, + # Where the crawl started. + source:, + # RE2 pattern URLs had to match to be crawled. Null when the crawl set none. + url_pattern: + ) + end + + sig do + override.returns( + { + follow_subdomains: T::Boolean, + max_depth: T.nilable(Integer), + max_pages: Integer, + source: ContextDev::CrawlControls::Source::Variants, + url_pattern: T.nilable(String) + } + ) + end + def to_hash + end + + # Where the crawl started. + module Source + extend ContextDev::Internal::Type::Union + + Variants = + T.type_alias do + T.any( + ContextDev::CrawlControls::Source::UnionMember0, + ContextDev::CrawlControls::Source::UnionMember1 + ) + end + + class UnionMember0 < ContextDev::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any( + ContextDev::CrawlControls::Source::UnionMember0, + ContextDev::Internal::AnyHash + ) + end + + sig do + returns( + ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol + ) + end + attr_accessor :type + + # Page the crawl started from. + sig { returns(String) } + attr_accessor :url + + sig do + params( + type: + ContextDev::CrawlControls::Source::UnionMember0::Type::OrSymbol, + url: String + ).returns(T.attached_class) + end + def self.new( + type:, + # Page the crawl started from. + url: + ) + end + + sig do + override.returns( + { + type: + ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol, + url: String + } + ) + end + def to_hash + end + + module Type + extend ContextDev::Internal::Type::Enum + + TaggedSymbol = + T.type_alias do + T.all( + Symbol, + ContextDev::CrawlControls::Source::UnionMember0::Type + ) + end + OrSymbol = T.type_alias { T.any(Symbol, String) } + + START_URL = + T.let( + :start_url, + ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol + ) + + sig do + override.returns( + T::Array[ + ContextDev::CrawlControls::Source::UnionMember0::Type::TaggedSymbol + ] + ) + end + def self.values + end + end + end + + class UnionMember1 < ContextDev::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any( + ContextDev::CrawlControls::Source::UnionMember1, + ContextDev::Internal::AnyHash + ) + end + + # Domain whose sitemap supplied the pages. + sig { returns(String) } + attr_accessor :domain + + sig do + returns( + ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol + ) + end + attr_accessor :type + + sig do + params( + domain: String, + type: + ContextDev::CrawlControls::Source::UnionMember1::Type::OrSymbol + ).returns(T.attached_class) + end + def self.new( + # Domain whose sitemap supplied the pages. + domain:, + type: + ) + end + + sig do + override.returns( + { + domain: String, + type: + ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol + } + ) + end + def to_hash + end + + module Type + extend ContextDev::Internal::Type::Enum + + TaggedSymbol = + T.type_alias do + T.all( + Symbol, + ContextDev::CrawlControls::Source::UnionMember1::Type + ) + end + OrSymbol = T.type_alias { T.any(Symbol, String) } + + SITEMAP = + T.let( + :sitemap, + ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol + ) + + sig do + override.returns( + T::Array[ + ContextDev::CrawlControls::Source::UnionMember1::Type::TaggedSymbol + ] + ) + end + def self.values + end + end + end + + sig do + override.returns( + T::Array[ContextDev::CrawlControls::Source::Variants] + ) + end + def self.variants + end + end + end + end +end diff --git a/rbi/context_dev/models/error.rbi b/rbi/context_dev/models/failure.rbi similarity index 51% rename from rbi/context_dev/models/error.rbi rename to rbi/context_dev/models/failure.rbi index 6f0ce137..8af7f3d6 100644 --- a/rbi/context_dev/models/error.rbi +++ b/rbi/context_dev/models/failure.rbi @@ -2,24 +2,27 @@ module ContextDev module Models - class Error < ContextDev::Internal::Type::BaseModel + class Failure < ContextDev::Internal::Type::BaseModel OrHash = - T.type_alias { T.any(ContextDev::Error, ContextDev::Internal::AnyHash) } + T.type_alias do + T.any(ContextDev::Failure, ContextDev::Internal::AnyHash) + end - # Batch error code. + # Why the batch itself stopped. sig { returns(String) } attr_accessor :code - # Batch error message. + # Human-readable explanation. sig { returns(String) } attr_accessor :message - # Why the batch failed. + # A failure of the batch as a whole, distinct from the per-page failures in + # `page_errors`. sig { params(code: String, message: String).returns(T.attached_class) } def self.new( - # Batch error code. + # Why the batch itself stopped. code:, - # Batch error message. + # Human-readable explanation. message: ) end diff --git a/rbi/context_dev/models/intake.rbi b/rbi/context_dev/models/intake.rbi new file mode 100644 index 00000000..c75130f8 --- /dev/null +++ b/rbi/context_dev/models/intake.rbi @@ -0,0 +1,88 @@ +# typed: strong + +module ContextDev + module Models + class Intake < ContextDev::Internal::Type::BaseModel + OrHash = + T.type_alias do + T.any(ContextDev::Intake, ContextDev::Internal::AnyHash) + end + + # URLs dropped before reserving because another entry resolved to the same page. + # Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than + # once. + sig { returns(Integer) } + attr_accessor :duplicates + + # URLs from your list rejected as unusable; the same ones are itemised in + # `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable + # page is rejected outright with a 400 rather than accepted with an empty list. + sig { returns(T.nilable(Integer)) } + attr_accessor :invalid + + # Pages credits were reserved for. Everything else — progress, the refund, the + # completion percentage — is measured against this. + sig { returns(Integer) } + attr_accessor :reserved + + # Whether `reserved` is an upper bound the batch may finish under. True only for a + # crawl that follows links, whose reachable page count is unknowable until it + # runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact + # page count. + sig { returns(T::Boolean) } + attr_accessor :reserved_is_ceiling + + # URLs in the list you sent, before validation and de-duplication. Null for a + # crawl, which is given a source rather than a list. + sig { returns(T.nilable(Integer)) } + attr_accessor :submitted + + # What submission took in, and what it charged for. + sig do + params( + duplicates: Integer, + invalid: T.nilable(Integer), + reserved: Integer, + reserved_is_ceiling: T::Boolean, + submitted: T.nilable(Integer) + ).returns(T.attached_class) + end + def self.new( + # URLs dropped before reserving because another entry resolved to the same page. + # Non-zero for sitemap crawls too, whose sitemaps routinely list a page more than + # once. + duplicates:, + # URLs from your list rejected as unusable; the same ones are itemised in + # `invalid_urls` at submission. Null for a crawl — a crawl that resolves no usable + # page is rejected outright with a 400 rather than accepted with an empty list. + invalid:, + # Pages credits were reserved for. Everything else — progress, the refund, the + # completion percentage — is measured against this. + reserved:, + # Whether `reserved` is an upper bound the batch may finish under. True only for a + # crawl that follows links, whose reachable page count is unknowable until it + # runs. False for a scrape and for a sitemap crawl, where `reserved` is an exact + # page count. + reserved_is_ceiling:, + # URLs in the list you sent, before validation and de-duplication. Null for a + # crawl, which is given a source rather than a list. + submitted: + ) + end + + sig do + override.returns( + { + duplicates: Integer, + invalid: T.nilable(Integer), + reserved: Integer, + reserved_is_ceiling: T::Boolean, + submitted: T.nilable(Integer) + } + ) + end + def to_hash + end + end + end +end diff --git a/rbi/context_dev/models/error_count.rbi b/rbi/context_dev/models/page_error_count.rbi similarity index 83% rename from rbi/context_dev/models/error_count.rbi rename to rbi/context_dev/models/page_error_count.rbi index a9c70143..65cf846d 100644 --- a/rbi/context_dev/models/error_count.rbi +++ b/rbi/context_dev/models/page_error_count.rbi @@ -2,10 +2,10 @@ module ContextDev module Models - class ErrorCount < ContextDev::Internal::Type::BaseModel + class PageErrorCount < ContextDev::Internal::Type::BaseModel OrHash = T.type_alias do - T.any(ContextDev::ErrorCount, ContextDev::Internal::AnyHash) + T.any(ContextDev::PageErrorCount, ContextDev::Internal::AnyHash) end # Error code for these failures. diff --git a/rbi/context_dev/models/web_screenshot_params.rbi b/rbi/context_dev/models/web_screenshot_params.rbi index 2e8c9ff2..1cd42779 100644 --- a/rbi/context_dev/models/web_screenshot_params.rbi +++ b/rbi/context_dev/models/web_screenshot_params.rbi @@ -27,9 +27,8 @@ module ContextDev end attr_writer :color_scheme - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). sig do returns(T.nilable(ContextDev::WebScreenshotParams::Country::OrSymbol)) end @@ -201,9 +200,8 @@ module ContextDev # Optional parameter to choose the site's visual theme in the screenshot. Use # 'light' or 'dark' when the site offers both appearances. color_scheme: nil, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # A specific URL to screenshot directly, bypassing domain resolution (e.g., # 'https://example.com/pricing'). When provided, the screenshot is taken of this @@ -322,9 +320,8 @@ module ContextDev end end - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/rbi/context_dev/models/web_web_crawl_md_params.rbi b/rbi/context_dev/models/web_web_crawl_md_params.rbi index baa72ffc..abfdd35c 100644 --- a/rbi/context_dev/models/web_web_crawl_md_params.rbi +++ b/rbi/context_dev/models/web_web_crawl_md_params.rbi @@ -15,9 +15,8 @@ module ContextDev sig { returns(String) } attr_accessor :url - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). sig do returns(T.nilable(ContextDev::WebWebCrawlMdParams::Country::OrSymbol)) end @@ -213,9 +212,8 @@ module ContextDev def self.new( # The starting URL for the crawl (must include http:// or https:// protocol) url:, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # CSS selectors to remove before each crawled page is converted to Markdown. # Applied after includeSelectors. Exclusion takes precedence: an element matching @@ -313,9 +311,8 @@ module ContextDev def to_hash end - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/rbi/context_dev/models/web_web_scrape_html_params.rbi b/rbi/context_dev/models/web_web_scrape_html_params.rbi index 263f7d6a..d1a33a2f 100644 --- a/rbi/context_dev/models/web_web_scrape_html_params.rbi +++ b/rbi/context_dev/models/web_web_scrape_html_params.rbi @@ -35,9 +35,8 @@ module ContextDev end attr_accessor :actions - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). sig do returns( T.nilable(ContextDev::WebWebScrapeHTMLParams::Country::OrSymbol) @@ -246,9 +245,8 @@ module ContextDev # content is captured. Requires a paid plan. Send a JSON array in the query # parameter. Maximum: 5 actions. actions: nil, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # CSS selectors to remove from the result. Applied after includeSelectors. # Exclusion takes precedence: an element matching both is removed. Examples: @@ -417,9 +415,8 @@ module ContextDev end end - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/rbi/context_dev/models/web_web_scrape_md_params.rbi b/rbi/context_dev/models/web_web_scrape_md_params.rbi index bc3998d6..e7e3046a 100644 --- a/rbi/context_dev/models/web_web_scrape_md_params.rbi +++ b/rbi/context_dev/models/web_web_scrape_md_params.rbi @@ -33,9 +33,8 @@ module ContextDev end attr_accessor :actions - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). sig do returns(T.nilable(ContextDev::WebWebScrapeMdParams::Country::OrSymbol)) end @@ -328,9 +327,8 @@ module ContextDev # content is captured. Requires a paid plan. Send a JSON array in the query # parameter. Maximum: 5 actions. actions: nil, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # CSS selectors to remove before conversion to Markdown. Applied after # includeSelectors. Exclusion takes precedence: an element matching both is @@ -520,9 +518,8 @@ module ContextDev end end - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). module Country extend ContextDev::Internal::Type::Enum diff --git a/rbi/context_dev/resources/batch.rbi b/rbi/context_dev/resources/batch.rbi index d553f903..78000f9a 100644 --- a/rbi/context_dev/resources/batch.rbi +++ b/rbi/context_dev/resources/batch.rbi @@ -3,9 +3,7 @@ module ContextDev module Resources class Batch - # Check progress and get download links when the batch finishes. Also returns the - # rejected-URL list and webhook signing secret from submission, so nothing is lost - # if the submit response was dropped. + # Check progress, and get download links once the batch finishes. sig do params( batch_id: String, @@ -66,9 +64,8 @@ module ContextDev ) end - # Page through the result records of a finished batch as JSON, in the same order - # as the downloadable result files. Use this instead of downloading and parsing - # the NDJSON files yourself. + # Page through a finished batch's results as JSON instead of downloading the + # NDJSON files. sig do params( batch_id: String, diff --git a/rbi/context_dev/resources/web.rbi b/rbi/context_dev/resources/web.rbi index 9c1a6cb1..5b32418a 100644 --- a/rbi/context_dev/resources/web.rbi +++ b/rbi/context_dev/resources/web.rbi @@ -216,9 +216,8 @@ module ContextDev # Optional parameter to choose the site's visual theme in the screenshot. Use # 'light' or 'dark' when the site offers both appearances. color_scheme: nil, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # A specific URL to screenshot directly, bypassing domain resolution (e.g., # 'https://example.com/pricing'). When provided, the screenshot is taken of this @@ -355,9 +354,8 @@ module ContextDev def web_crawl_md( # The starting URL for the crawl (must include http:// or https:// protocol) url:, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # CSS selectors to remove before each crawled page is converted to Markdown. # Applied after includeSelectors. Exclusion takes precedence: an element matching @@ -473,9 +471,8 @@ module ContextDev # content is captured. Requires a paid plan. Send a JSON array in the query # parameter. Maximum: 5 actions. actions: nil, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # CSS selectors to remove from the result. Applied after includeSelectors. # Exclusion takes precedence: an element matching both is removed. Examples: @@ -673,9 +670,8 @@ module ContextDev # content is captured. Requires a paid plan. Send a JSON array in the query # parameter. Maximum: 5 actions. actions: nil, - # Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev - # residential proxy exit location. Must be one of Context.dev's supported - # countries. When provided, Context.dev fetches the target page from that country. + # Fetch the target page through a residential proxy in this country (ISO 3166-1 + # alpha-2). country: nil, # CSS selectors to remove before conversion to Markdown. Applied after # includeSelectors. Exclusion takes precedence: an element matching both is diff --git a/sig/context_dev/models.rbs b/sig/context_dev/models.rbs index 66ce2bb6..47a38e2c 100644 --- a/sig/context_dev/models.rbs +++ b/sig/context_dev/models.rbs @@ -17,14 +17,16 @@ module ContextDev class BrandRetrieveSimplifiedParams = ContextDev::Models::BrandRetrieveSimplifiedParams - class Error = ContextDev::Models::Error + class CrawlControls = ContextDev::Models::CrawlControls - class ErrorCount = ContextDev::Models::ErrorCount + class Failure = ContextDev::Models::Failure class IndustryRetrieveNaicsParams = ContextDev::Models::IndustryRetrieveNaicsParams class IndustryRetrieveSicParams = ContextDev::Models::IndustryRetrieveSicParams + class Intake = ContextDev::Models::Intake + class MonitorCreateParams = ContextDev::Models::MonitorCreateParams class MonitorDeleteParams = ContextDev::Models::MonitorDeleteParams @@ -51,6 +53,8 @@ module ContextDev class MonitorUpdateParams = ContextDev::Models::MonitorUpdateParams + class PageErrorCount = ContextDev::Models::PageErrorCount + class ParseHandleParams = ContextDev::Models::ParseHandleParams class UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams diff --git a/sig/context_dev/models/batch_cancel_response.rbs b/sig/context_dev/models/batch_cancel_response.rbs index 5bdf7889..e557398d 100644 --- a/sig/context_dev/models/batch_cancel_response.rbs +++ b/sig/context_dev/models/batch_cancel_response.rbs @@ -3,36 +3,35 @@ module ContextDev type batch_cancel_response = { id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchCancelResponse::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchCancelResponse::Input, + format_: ContextDev::Models::BatchCancelResponse::format_, + input: ContextDev::Intake, mode: ContextDev::Models::BatchCancelResponse::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchCancelResponse::Progress, - results: ContextDev::Models::BatchCancelResponse::Results?, status: ContextDev::Models::BatchCancelResponse::status, tags: ::Array[String], timing: ContextDev::Models::BatchCancelResponse::Timing, - type: ContextDev::Models::BatchCancelResponse::type_, key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata } class BatchCancelResponse < ContextDev::Internal::Type::BaseModel attr_accessor id: String - attr_accessor credits: ContextDev::Models::BatchCancelResponse::Credits + attr_accessor crawl: ContextDev::CrawlControls? - attr_accessor error: ContextDev::Error? + attr_accessor credits: ContextDev::Models::BatchCancelResponse::Credits - attr_accessor errors: ::Array[ContextDev::ErrorCount] + attr_accessor format_: ContextDev::Models::BatchCancelResponse::format_ - attr_accessor input: ContextDev::Models::BatchCancelResponse::Input + attr_accessor input: ContextDev::Intake attr_accessor mode: ContextDev::Models::BatchCancelResponse::mode - attr_accessor progress: ContextDev::Models::BatchCancelResponse::Progress + attr_accessor page_errors: ::Array[ContextDev::PageErrorCount] - attr_accessor results: ContextDev::Models::BatchCancelResponse::Results? + attr_accessor progress: ContextDev::Models::BatchCancelResponse::Progress attr_accessor status: ContextDev::Models::BatchCancelResponse::status @@ -40,8 +39,6 @@ module ContextDev attr_accessor timing: ContextDev::Models::BatchCancelResponse::Timing - attr_accessor type: ContextDev::Models::BatchCancelResponse::type_ - attr_reader key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata? def key_metadata=: ( @@ -50,78 +47,53 @@ module ContextDev def initialize: ( id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchCancelResponse::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchCancelResponse::Input, + format_: ContextDev::Models::BatchCancelResponse::format_, + input: ContextDev::Intake, mode: ContextDev::Models::BatchCancelResponse::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchCancelResponse::Progress, - results: ContextDev::Models::BatchCancelResponse::Results?, status: ContextDev::Models::BatchCancelResponse::status, tags: ::Array[String], timing: ContextDev::Models::BatchCancelResponse::Timing, - type: ContextDev::Models::BatchCancelResponse::type_, ?key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata ) -> void def to_hash: -> { id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchCancelResponse::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchCancelResponse::Input, + format_: ContextDev::Models::BatchCancelResponse::format_, + input: ContextDev::Intake, mode: ContextDev::Models::BatchCancelResponse::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchCancelResponse::Progress, - results: ContextDev::Models::BatchCancelResponse::Results?, status: ContextDev::Models::BatchCancelResponse::status, tags: ::Array[String], timing: ContextDev::Models::BatchCancelResponse::Timing, - type: ContextDev::Models::BatchCancelResponse::type_, key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata } - type credits = { charged: Integer, estimated: Integer } + type credits = { reserved: Integer } class Credits < ContextDev::Internal::Type::BaseModel - attr_accessor charged: Integer - - attr_accessor estimated: Integer + attr_accessor reserved: Integer - def initialize: (charged: Integer, estimated: Integer) -> void + def initialize: (reserved: Integer) -> void - def to_hash: -> { charged: Integer, estimated: Integer } + def to_hash: -> { reserved: Integer } end - type input = - { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } - - class Input < ContextDev::Internal::Type::BaseModel - attr_accessor accepted: Integer - - attr_accessor duplicates: Integer + type format_ = :markdown | :html - attr_accessor invalid: Integer - - attr_accessor submitted: Integer + module Format + extend ContextDev::Internal::Type::Enum - def initialize: ( - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - ) -> void + MARKDOWN: :markdown + HTML: :html - def to_hash: -> { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } + def self?.values: -> ::Array[ContextDev::Models::BatchCancelResponse::format_] end type mode = :scrape | :crawl @@ -157,90 +129,26 @@ module ContextDev } end - type results = - { - expires_at: String, - files: ::Array[ContextDev::Models::BatchCancelResponse::Results::File] - } - - class Results < ContextDev::Internal::Type::BaseModel - attr_accessor expires_at: String - - attr_accessor files: ::Array[ContextDev::Models::BatchCancelResponse::Results::File] - - def initialize: ( - expires_at: String, - files: ::Array[ContextDev::Models::BatchCancelResponse::Results::File] - ) -> void - - def to_hash: -> { - expires_at: String, - files: ::Array[ContextDev::Models::BatchCancelResponse::Results::File] - } - - type file = { bytes: Integer, items: Integer, url: String } - - class File < ContextDev::Internal::Type::BaseModel - attr_accessor bytes: Integer - - attr_accessor items: Integer - - attr_accessor url: String - - def initialize: (bytes: Integer, items: Integer, url: String) -> void - - def to_hash: -> { bytes: Integer, items: Integer, url: String } - end - end - - type status = - :queued | :running | :cancelling | :completed | :cancelled | :failed + type status = :cancelling module Status extend ContextDev::Internal::Type::Enum - QUEUED: :queued - RUNNING: :running CANCELLING: :cancelling - COMPLETED: :completed - CANCELLED: :cancelled - FAILED: :failed def self?.values: -> ::Array[ContextDev::Models::BatchCancelResponse::status] end - type timing = - { completed_at: String?, created_at: String, started_at: String? } + type timing = { created_at: String, started_at: String? } class Timing < ContextDev::Internal::Type::BaseModel - attr_accessor completed_at: String? - attr_accessor created_at: String attr_accessor started_at: String? - def initialize: ( - completed_at: String?, - created_at: String, - started_at: String? - ) -> void - - def to_hash: -> { - completed_at: String?, - created_at: String, - started_at: String? - } - end - - type type_ = :markdown | :html - - module Type - extend ContextDev::Internal::Type::Enum - - MARKDOWN: :markdown - HTML: :html + def initialize: (created_at: String, started_at: String?) -> void - def self?.values: -> ::Array[ContextDev::Models::BatchCancelResponse::type_] + def to_hash: -> { created_at: String, started_at: String? } end type key_metadata = diff --git a/sig/context_dev/models/batch_list_response.rbs b/sig/context_dev/models/batch_list_response.rbs index 571de08e..1506ff2f 100644 --- a/sig/context_dev/models/batch_list_response.rbs +++ b/sig/context_dev/models/batch_list_response.rbs @@ -44,32 +44,37 @@ module ContextDev type data = { id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchListResponse::Data::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchListResponse::Data::Input, + failure: ContextDev::Failure?, + format_: ContextDev::Models::BatchListResponse::Data::format_, + input: ContextDev::Intake, mode: ContextDev::Models::BatchListResponse::Data::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchListResponse::Data::Progress, results: ContextDev::Models::BatchListResponse::Data::Results?, status: ContextDev::Models::BatchListResponse::Data::status, tags: ::Array[String], - timing: ContextDev::Models::BatchListResponse::Data::Timing, - type: ContextDev::Models::BatchListResponse::Data::type_ + timing: ContextDev::Models::BatchListResponse::Data::Timing } class Data < ContextDev::Internal::Type::BaseModel attr_accessor id: String + attr_accessor crawl: ContextDev::CrawlControls? + attr_accessor credits: ContextDev::Models::BatchListResponse::Data::Credits - attr_accessor error: ContextDev::Error? + attr_accessor failure: ContextDev::Failure? - attr_accessor errors: ::Array[ContextDev::ErrorCount] + attr_accessor format_: ContextDev::Models::BatchListResponse::Data::format_ - attr_accessor input: ContextDev::Models::BatchListResponse::Data::Input + attr_accessor input: ContextDev::Intake attr_accessor mode: ContextDev::Models::BatchListResponse::Data::mode + attr_accessor page_errors: ::Array[ContextDev::PageErrorCount] + attr_accessor progress: ContextDev::Models::BatchListResponse::Data::Progress attr_accessor results: ContextDev::Models::BatchListResponse::Data::Results? @@ -80,80 +85,65 @@ module ContextDev attr_accessor timing: ContextDev::Models::BatchListResponse::Data::Timing - attr_accessor type: ContextDev::Models::BatchListResponse::Data::type_ - def initialize: ( id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchListResponse::Data::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchListResponse::Data::Input, + failure: ContextDev::Failure?, + format_: ContextDev::Models::BatchListResponse::Data::format_, + input: ContextDev::Intake, mode: ContextDev::Models::BatchListResponse::Data::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchListResponse::Data::Progress, results: ContextDev::Models::BatchListResponse::Data::Results?, status: ContextDev::Models::BatchListResponse::Data::status, tags: ::Array[String], - timing: ContextDev::Models::BatchListResponse::Data::Timing, - type: ContextDev::Models::BatchListResponse::Data::type_ + timing: ContextDev::Models::BatchListResponse::Data::Timing ) -> void def to_hash: -> { id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchListResponse::Data::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchListResponse::Data::Input, + failure: ContextDev::Failure?, + format_: ContextDev::Models::BatchListResponse::Data::format_, + input: ContextDev::Intake, mode: ContextDev::Models::BatchListResponse::Data::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchListResponse::Data::Progress, results: ContextDev::Models::BatchListResponse::Data::Results?, status: ContextDev::Models::BatchListResponse::Data::status, tags: ::Array[String], - timing: ContextDev::Models::BatchListResponse::Data::Timing, - type: ContextDev::Models::BatchListResponse::Data::type_ + timing: ContextDev::Models::BatchListResponse::Data::Timing } - type credits = { charged: Integer, estimated: Integer } + type credits = { net: Integer, refunded: Integer, reserved: Integer } class Credits < ContextDev::Internal::Type::BaseModel - attr_accessor charged: Integer + attr_accessor net: Integer - attr_accessor estimated: Integer + attr_accessor refunded: Integer - def initialize: (charged: Integer, estimated: Integer) -> void + attr_accessor reserved: Integer - def to_hash: -> { charged: Integer, estimated: Integer } - end - - type input = - { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } - - class Input < ContextDev::Internal::Type::BaseModel - attr_accessor accepted: Integer + def initialize: ( + net: Integer, + refunded: Integer, + reserved: Integer + ) -> void - attr_accessor duplicates: Integer + def to_hash: -> { net: Integer, refunded: Integer, reserved: Integer } + end - attr_accessor invalid: Integer + type format_ = :markdown | :html - attr_accessor submitted: Integer + module Format + extend ContextDev::Internal::Type::Enum - def initialize: ( - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - ) -> void + MARKDOWN: :markdown + HTML: :html - def to_hash: -> { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } + def self?.values: -> ::Array[ContextDev::Models::BatchListResponse::Data::format_] end type mode = :scrape | :crawl @@ -268,17 +258,6 @@ module ContextDev started_at: String? } end - - type type_ = :markdown | :html - - module Type - extend ContextDev::Internal::Type::Enum - - MARKDOWN: :markdown - HTML: :html - - def self?.values: -> ::Array[ContextDev::Models::BatchListResponse::Data::type_] - end end type key_metadata = diff --git a/sig/context_dev/models/batch_retrieve_response.rbs b/sig/context_dev/models/batch_retrieve_response.rbs index 654bdcad..6f6f136e 100644 --- a/sig/context_dev/models/batch_retrieve_response.rbs +++ b/sig/context_dev/models/batch_retrieve_response.rbs @@ -3,37 +3,41 @@ module ContextDev type batch_retrieve_response = { id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchRetrieveResponse::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchRetrieveResponse::Input, + failure: ContextDev::Failure?, + format_: ContextDev::Models::BatchRetrieveResponse::format_, + input: ContextDev::Intake, invalid_urls: ::Array[ContextDev::Models::BatchRetrieveResponse::InvalidURL], mode: ContextDev::Models::BatchRetrieveResponse::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchRetrieveResponse::Progress, results: ContextDev::Models::BatchRetrieveResponse::Results?, status: ContextDev::Models::BatchRetrieveResponse::status, tags: ::Array[String], timing: ContextDev::Models::BatchRetrieveResponse::Timing, - type: ContextDev::Models::BatchRetrieveResponse::type_, - key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata, - webhook_secret: String + key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata } class BatchRetrieveResponse < ContextDev::Internal::Type::BaseModel attr_accessor id: String + attr_accessor crawl: ContextDev::CrawlControls? + attr_accessor credits: ContextDev::Models::BatchRetrieveResponse::Credits - attr_accessor error: ContextDev::Error? + attr_accessor failure: ContextDev::Failure? - attr_accessor errors: ::Array[ContextDev::ErrorCount] + attr_accessor format_: ContextDev::Models::BatchRetrieveResponse::format_ - attr_accessor input: ContextDev::Models::BatchRetrieveResponse::Input + attr_accessor input: ContextDev::Intake attr_accessor invalid_urls: ::Array[ContextDev::Models::BatchRetrieveResponse::InvalidURL] attr_accessor mode: ContextDev::Models::BatchRetrieveResponse::mode + attr_accessor page_errors: ::Array[ContextDev::PageErrorCount] + attr_accessor progress: ContextDev::Models::BatchRetrieveResponse::Progress attr_accessor results: ContextDev::Models::BatchRetrieveResponse::Results? @@ -44,96 +48,75 @@ module ContextDev attr_accessor timing: ContextDev::Models::BatchRetrieveResponse::Timing - attr_accessor type: ContextDev::Models::BatchRetrieveResponse::type_ - attr_reader key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata? def key_metadata=: ( ContextDev::Models::BatchRetrieveResponse::KeyMetadata ) -> ContextDev::Models::BatchRetrieveResponse::KeyMetadata - attr_reader webhook_secret: String? - - def webhook_secret=: (String) -> String - def initialize: ( id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchRetrieveResponse::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchRetrieveResponse::Input, + failure: ContextDev::Failure?, + format_: ContextDev::Models::BatchRetrieveResponse::format_, + input: ContextDev::Intake, invalid_urls: ::Array[ContextDev::Models::BatchRetrieveResponse::InvalidURL], mode: ContextDev::Models::BatchRetrieveResponse::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchRetrieveResponse::Progress, results: ContextDev::Models::BatchRetrieveResponse::Results?, status: ContextDev::Models::BatchRetrieveResponse::status, tags: ::Array[String], timing: ContextDev::Models::BatchRetrieveResponse::Timing, - type: ContextDev::Models::BatchRetrieveResponse::type_, - ?key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata, - ?webhook_secret: String + ?key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata ) -> void def to_hash: -> { id: String, + crawl: ContextDev::CrawlControls?, credits: ContextDev::Models::BatchRetrieveResponse::Credits, - error: ContextDev::Error?, - errors: ::Array[ContextDev::ErrorCount], - input: ContextDev::Models::BatchRetrieveResponse::Input, + failure: ContextDev::Failure?, + format_: ContextDev::Models::BatchRetrieveResponse::format_, + input: ContextDev::Intake, invalid_urls: ::Array[ContextDev::Models::BatchRetrieveResponse::InvalidURL], mode: ContextDev::Models::BatchRetrieveResponse::mode, + page_errors: ::Array[ContextDev::PageErrorCount], progress: ContextDev::Models::BatchRetrieveResponse::Progress, results: ContextDev::Models::BatchRetrieveResponse::Results?, status: ContextDev::Models::BatchRetrieveResponse::status, tags: ::Array[String], timing: ContextDev::Models::BatchRetrieveResponse::Timing, - type: ContextDev::Models::BatchRetrieveResponse::type_, - key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata, - webhook_secret: String + key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata } - type credits = { charged: Integer, estimated: Integer } + type credits = { net: Integer, refunded: Integer, reserved: Integer } class Credits < ContextDev::Internal::Type::BaseModel - attr_accessor charged: Integer - - attr_accessor estimated: Integer - - def initialize: (charged: Integer, estimated: Integer) -> void + attr_accessor net: Integer - def to_hash: -> { charged: Integer, estimated: Integer } - end + attr_accessor refunded: Integer - type input = - { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } + attr_accessor reserved: Integer - class Input < ContextDev::Internal::Type::BaseModel - attr_accessor accepted: Integer + def initialize: ( + net: Integer, + refunded: Integer, + reserved: Integer + ) -> void - attr_accessor duplicates: Integer + def to_hash: -> { net: Integer, refunded: Integer, reserved: Integer } + end - attr_accessor invalid: Integer + type format_ = :markdown | :html - attr_accessor submitted: Integer + module Format + extend ContextDev::Internal::Type::Enum - def initialize: ( - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - ) -> void + MARKDOWN: :markdown + HTML: :html - def to_hash: -> { - accepted: Integer, - duplicates: Integer, - invalid: Integer, - submitted: Integer - } + def self?.values: -> ::Array[ContextDev::Models::BatchRetrieveResponse::format_] end type invalid_url = { reason: String, url: String } @@ -256,17 +239,6 @@ module ContextDev } end - type type_ = :markdown | :html - - module Type - extend ContextDev::Internal::Type::Enum - - MARKDOWN: :markdown - HTML: :html - - def self?.values: -> ::Array[ContextDev::Models::BatchRetrieveResponse::type_] - end - type key_metadata = { credits_consumed: Integer, credits_remaining: Integer } diff --git a/sig/context_dev/models/crawl_controls.rbs b/sig/context_dev/models/crawl_controls.rbs new file mode 100644 index 00000000..943a274b --- /dev/null +++ b/sig/context_dev/models/crawl_controls.rbs @@ -0,0 +1,114 @@ +module ContextDev + module Models + type crawl_controls = + { + follow_subdomains: bool, + max_depth: Integer?, + max_pages: Integer, + source: ContextDev::Models::CrawlControls::source, + url_pattern: String? + } + + class CrawlControls < ContextDev::Internal::Type::BaseModel + attr_accessor follow_subdomains: bool + + attr_accessor max_depth: Integer? + + attr_accessor max_pages: Integer + + attr_accessor source: ContextDev::Models::CrawlControls::source + + attr_accessor url_pattern: String? + + def initialize: ( + follow_subdomains: bool, + max_depth: Integer?, + max_pages: Integer, + source: ContextDev::Models::CrawlControls::source, + url_pattern: String? + ) -> void + + def to_hash: -> { + follow_subdomains: bool, + max_depth: Integer?, + max_pages: Integer, + source: ContextDev::Models::CrawlControls::source, + url_pattern: String? + } + + type source = + ContextDev::CrawlControls::Source::UnionMember0 + | ContextDev::CrawlControls::Source::UnionMember1 + + module Source + extend ContextDev::Internal::Type::Union + + type union_member0 = + { + type: ContextDev::Models::CrawlControls::Source::UnionMember0::type_, + url: String + } + + class UnionMember0 < ContextDev::Internal::Type::BaseModel + attr_accessor type: ContextDev::Models::CrawlControls::Source::UnionMember0::type_ + + attr_accessor url: String + + def initialize: ( + type: ContextDev::Models::CrawlControls::Source::UnionMember0::type_, + url: String + ) -> void + + def to_hash: -> { + type: ContextDev::Models::CrawlControls::Source::UnionMember0::type_, + url: String + } + + type type_ = :start_url + + module Type + extend ContextDev::Internal::Type::Enum + + START_URL: :start_url + + def self?.values: -> ::Array[ContextDev::Models::CrawlControls::Source::UnionMember0::type_] + end + end + + type union_member1 = + { + domain: String, + type: ContextDev::Models::CrawlControls::Source::UnionMember1::type_ + } + + class UnionMember1 < ContextDev::Internal::Type::BaseModel + attr_accessor domain: String + + attr_accessor type: ContextDev::Models::CrawlControls::Source::UnionMember1::type_ + + def initialize: ( + domain: String, + type: ContextDev::Models::CrawlControls::Source::UnionMember1::type_ + ) -> void + + def to_hash: -> { + domain: String, + type: ContextDev::Models::CrawlControls::Source::UnionMember1::type_ + } + + type type_ = :sitemap + + module Type + extend ContextDev::Internal::Type::Enum + + SITEMAP: :sitemap + + def self?.values: -> ::Array[ContextDev::Models::CrawlControls::Source::UnionMember1::type_] + end + end + + def self?.variants: -> ::Array[ContextDev::Models::CrawlControls::source] + end + end + end +end diff --git a/sig/context_dev/models/error.rbs b/sig/context_dev/models/failure.rbs similarity index 68% rename from sig/context_dev/models/error.rbs rename to sig/context_dev/models/failure.rbs index 67829209..cf0c83c6 100644 --- a/sig/context_dev/models/error.rbs +++ b/sig/context_dev/models/failure.rbs @@ -1,8 +1,8 @@ module ContextDev module Models - type error = { code: String, message: String } + type failure = { code: String, message: String } - class Error < ContextDev::Internal::Type::BaseModel + class Failure < ContextDev::Internal::Type::BaseModel attr_accessor code: String attr_accessor message: String diff --git a/sig/context_dev/models/intake.rbs b/sig/context_dev/models/intake.rbs new file mode 100644 index 00000000..96800564 --- /dev/null +++ b/sig/context_dev/models/intake.rbs @@ -0,0 +1,40 @@ +module ContextDev + module Models + type intake = + { + duplicates: Integer, + invalid: Integer?, + reserved: Integer, + reserved_is_ceiling: bool, + submitted: Integer? + } + + class Intake < ContextDev::Internal::Type::BaseModel + attr_accessor duplicates: Integer + + attr_accessor invalid: Integer? + + attr_accessor reserved: Integer + + attr_accessor reserved_is_ceiling: bool + + attr_accessor submitted: Integer? + + def initialize: ( + duplicates: Integer, + invalid: Integer?, + reserved: Integer, + reserved_is_ceiling: bool, + submitted: Integer? + ) -> void + + def to_hash: -> { + duplicates: Integer, + invalid: Integer?, + reserved: Integer, + reserved_is_ceiling: bool, + submitted: Integer? + } + end + end +end diff --git a/sig/context_dev/models/error_count.rbs b/sig/context_dev/models/page_error_count.rbs similarity index 65% rename from sig/context_dev/models/error_count.rbs rename to sig/context_dev/models/page_error_count.rbs index b0f2bf0a..0740b6b5 100644 --- a/sig/context_dev/models/error_count.rbs +++ b/sig/context_dev/models/page_error_count.rbs @@ -1,8 +1,8 @@ module ContextDev module Models - type error_count = { code: String, count: Integer } + type page_error_count = { code: String, count: Integer } - class ErrorCount < ContextDev::Internal::Type::BaseModel + class PageErrorCount < ContextDev::Internal::Type::BaseModel attr_accessor code: String attr_accessor count: Integer diff --git a/test/context_dev/resources/batch_test.rb b/test/context_dev/resources/batch_test.rb index 1e5fb5de..dbdead45 100644 --- a/test/context_dev/resources/batch_test.rb +++ b/test/context_dev/resources/batch_test.rb @@ -15,20 +15,20 @@ def test_retrieve assert_pattern do response => { id: String, + crawl: ContextDev::CrawlControls | nil, credits: ContextDev::Models::BatchRetrieveResponse::Credits, - error: ContextDev::Error | nil, - errors: ^(ContextDev::Internal::Type::ArrayOf[ContextDev::ErrorCount]), - input: ContextDev::Models::BatchRetrieveResponse::Input, + failure: ContextDev::Failure | nil, + format_: ContextDev::Models::BatchRetrieveResponse::Format, + input: ContextDev::Intake, invalid_urls: ^(ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchRetrieveResponse::InvalidURL]), mode: ContextDev::Models::BatchRetrieveResponse::Mode, + page_errors: ^(ContextDev::Internal::Type::ArrayOf[ContextDev::PageErrorCount]), progress: ContextDev::Models::BatchRetrieveResponse::Progress, results: ContextDev::Models::BatchRetrieveResponse::Results | nil, status: ContextDev::Models::BatchRetrieveResponse::Status, tags: ^(ContextDev::Internal::Type::ArrayOf[String]), timing: ContextDev::Models::BatchRetrieveResponse::Timing, - type: ContextDev::Models::BatchRetrieveResponse::Type, - key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata | nil, - webhook_secret: String | nil + key_metadata: ContextDev::Models::BatchRetrieveResponse::KeyMetadata | nil } end end @@ -64,17 +64,16 @@ def test_cancel assert_pattern do response => { id: String, + crawl: ContextDev::CrawlControls | nil, credits: ContextDev::Models::BatchCancelResponse::Credits, - error: ContextDev::Error | nil, - errors: ^(ContextDev::Internal::Type::ArrayOf[ContextDev::ErrorCount]), - input: ContextDev::Models::BatchCancelResponse::Input, + format_: ContextDev::Models::BatchCancelResponse::Format, + input: ContextDev::Intake, mode: ContextDev::Models::BatchCancelResponse::Mode, + page_errors: ^(ContextDev::Internal::Type::ArrayOf[ContextDev::PageErrorCount]), progress: ContextDev::Models::BatchCancelResponse::Progress, - results: ContextDev::Models::BatchCancelResponse::Results | nil, status: ContextDev::Models::BatchCancelResponse::Status, tags: ^(ContextDev::Internal::Type::ArrayOf[String]), timing: ContextDev::Models::BatchCancelResponse::Timing, - type: ContextDev::Models::BatchCancelResponse::Type, key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata | nil } end