Class: Enconvert::V2

Inherits:
Object
  • Object
show all
Includes:
Internal
Defined in:
lib/enconvert/v2.rb,
lib/enconvert/v2_results.rb

Defined Under Namespace

Modules: Enums Classes: DiscoverResult, DistillItem, DistillResult, IngestJob, IngestJobList, IngestJobSummary, LookupItem, LookupResult, PerceiveBatchResult, PerceiveDirectResult, PerceiveResult, V2OutputArtifact, V2Tokens, Watcher, WatcherList, WatcherSnapshot, WatcherSnapshotList, WatcherSummary, WebhookRetryResult, WebhookSecret

Instance Method Summary collapse

Methods included from Internal

build_multipart, extract_error_message, new_job_id, raise_for_status, serialize_pdf_options, to_file_part

Constructor Details

#initialize(transport) ⇒ V2

Returns a new instance of V2.



26
27
28
 
# File 'lib/enconvert/v2.rb', line 26

def initialize(transport)
  @transport = transport
end

Instance Method Details

#cancel_ingest_job(job_id) ⇒ Object

Cancel a queued/processing ingest job. Idempotent: canceling an already-terminal job returns it unchanged.



279
280
281
 
# File 'lib/enconvert/v2.rb', line 279

def cancel_ingest_job(job_id)
  to_ingest_job(delete_request("/v2/ingest/#{encode_path(job_id)}"))
end

#create_watcher(url, frequency_minutes: nil, diff_mode: nil, track_fields: nil, webhook_url: nil, notify_email: nil) ⇒ Object

Create a watcher that re-renders url on a fixed cadence (hourly floor) and notifies on changes via email and/or webhook.



314
315
316
317
318
319
320
321
322
323
 
# File 'lib/enconvert/v2.rb', line 314

def create_watcher(url, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
                    webhook_url: nil, notify_email: nil)
  body = { url: url }
  body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
  body[:diff_mode] = diff_mode unless diff_mode.nil?
  body[:track_fields] = track_fields unless track_fields.nil?
  body[:webhook_url] = webhook_url unless webhook_url.nil?
  body[:notify_email] = notify_email unless notify_email.nil?
  to_watcher(post("/v2/watch", body))
end

#delete_watcher(watcher_id) ⇒ Object

Soft-delete a watcher (idempotent). Returns the tombstoned watcher with status "deleted".



373
374
375
 
# File 'lib/enconvert/v2.rb', line 373

def delete_watcher(watcher_id)
  to_watcher(delete_request("/v2/watch/#{encode_path(watcher_id)}"))
end

#discover(url, **opts) ⇒ Object

List a site's URLs via sitemap, HTTP crawl, or both. No browser rendering — fast and does not consume perceive quota.

opts accepts: mode, max_urls, max_depth, include_patterns, exclude_patterns, same_domain_only, respect_robots.



112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/enconvert/v2.rb', line 112

def discover(url, **opts)
  body = { url: url }
  body[:mode] = opts[:mode] if opts.key?(:mode)
  body[:max_urls] = opts[:max_urls] if opts.key?(:max_urls)
  body[:max_depth] = opts[:max_depth] if opts.key?(:max_depth)
  body[:include_patterns] = opts[:include_patterns] if opts.key?(:include_patterns)
  body[:exclude_patterns] = opts[:exclude_patterns] if opts.key?(:exclude_patterns)
  body[:same_domain_only] = opts[:same_domain_only] if opts.key?(:same_domain_only)
  body[:respect_robots] = opts[:respect_robots] if opts.key?(:respect_robots)
  to_discover_result(post("/v2/discover", body))
end

#distill(urls: nil, discover_from: nil, schema: nil, css_schema: nil, wait_for: nil, wait_timeout_ms: nil, headers: nil, cookies: nil, respect_robots: nil) ⇒ Object

Extract structured data matching schema from explicit URLs or from a discovered site. An optional css_schema answers fields for free; anything it misses escalates to the LLM tier (plan-gated).

Provide exactly one of urls (non-empty Array) or discover_from (Hash with :url, :mode, :max_pages). schema is required.

Raises:



158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/enconvert/v2.rb', line 158

def distill(urls: nil, discover_from: nil, schema: nil, css_schema: nil, wait_for: nil,
            wait_timeout_ms: nil, headers: nil, cookies: nil, respect_robots: nil)
  has_urls = urls.is_a?(Array) && !urls.empty?
  has_discover = !discover_from.nil?
  raise Enconvert::Error, "distill: provide exactly one of 'urls' or 'discover_from'" if has_urls == has_discover
  raise Enconvert::Error, "distill: 'schema' is required and must be a Hash" unless schema.is_a?(Hash)

  body = { schema: schema }
  body[:urls] = urls if has_urls
  if discover_from
    df = { url: discover_from[:url] }
    df[:mode] = discover_from[:mode] if discover_from.key?(:mode)
    df[:max_pages] = discover_from[:max_pages] if discover_from.key?(:max_pages)
    body[:discover_from] = df
  end
  body[:css_schema] = serialize_css_schema(css_schema) unless css_schema.nil?
  body[:wait_for] = wait_for unless wait_for.nil?
  body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
  body[:headers] = headers unless headers.nil?
  body[:cookies] = cookies unless cookies.nil?
  body[:respect_robots] = respect_robots unless respect_robots.nil?
  to_distill_result(post("/v2/distill", body))
end

#download_perceive_artifact(operation_id, output: nil) ⇒ Object

Download one stored artifact of an earlier perceive operation as raw bytes. output may be omitted when the operation produced exactly one artifact (400 otherwise, listing the available outputs). 410 once the artifact passed the plan's retention window.



81
82
83
84
85
86
87
 
# File 'lib/enconvert/v2.rb', line 81

def download_perceive_artifact(operation_id, output: nil)
  query = "?direct_download=true"
  query += "&output=#{encode_path(output)}" unless output.nil?
  resp = @transport.request(:get, "/v2/perceive/#{encode_path(operation_id)}#{query}")
  raise_for_status(resp)
  to_perceive_direct_result(resp)
end

#get_ingest_job(job_id) ⇒ Object

Get one ingest job by id (ing_...).



273
274
275
 
# File 'lib/enconvert/v2.rb', line 273

def get_ingest_job(job_id)
  to_ingest_job(get("/v2/ingest/#{encode_path(job_id)}"))
end

#get_perceive_batch(job_id) ⇒ Object

Poll a perceive batch by job_id. Items fill in as URLs complete.



99
100
101
 
# File 'lib/enconvert/v2.rb', line 99

def get_perceive_batch(job_id)
  to_perceive_batch_result(get("/v2/perceive/batch/#{encode_path(job_id)}"))
end

#get_perceive_operation(operation_id) ⇒ Object

Re-fetch a perceive operation by id (per_...). Artifact URLs are freshly re-signed on every call.



73
74
75
 
# File 'lib/enconvert/v2.rb', line 73

def get_perceive_operation(operation_id)
  to_perceive_result(get("/v2/perceive/#{encode_path(operation_id)}"))
end

#get_watcher(watcher_id) ⇒ Object

Get one watcher by id (wat_...). Deleted watchers read as 404.



338
339
340
 
# File 'lib/enconvert/v2.rb', line 338

def get_watcher(watcher_id)
  to_watcher(get("/v2/watch/#{encode_path(watcher_id)}"))
end

#get_watcher_snapshots(watcher_id, limit: nil) ⇒ Object

Page through a watcher's check history, newest first.



343
344
345
346
347
348
349
350
351
352
 
# File 'lib/enconvert/v2.rb', line 343

def get_watcher_snapshots(watcher_id, limit: nil)
  query = limit.nil? ? "" : "?limit=#{limit}"
  d = get("/v2/watch/#{encode_path(watcher_id)}/snapshots#{query}")
  snapshots = d["snapshots"].is_a?(Array) ? d["snapshots"] : []
  WatcherSnapshotList.new(
    watcher_id: str(d["watcher_id"]),
    snapshots: snapshots.map { |s| to_watcher_snapshot(s) },
    limit: num(d["limit"], 20)
  )
end

#get_webhook_secret ⇒ Object

Get (creating on first call) the project's webhook signing secret and the header/scheme details needed to verify deliveries.



298
299
300
 
# File 'lib/enconvert/v2.rb', line 298

def get_webhook_secret
  to_webhook_secret(get("/v2/ingest/webhook-secret"))
end

#ingest(mode: nil, url: nil, urls: nil, max_pages: nil, max_depth: nil, same_domain_only: nil, include_patterns: nil, exclude_patterns: nil, respect_robots: nil, wait_for: nil, wait_timeout_ms: nil, chunk: nil, webhook_url: nil) ⇒ Object

Start an ingest job: turn explicit URLs or a discovered site into chunked, RAG-ready JSONL. Always asynchronous — returns the queued job; poll get_ingest_job or configure webhook_url for completion.

mode defaults to "urls" for validation purposes but is only sent on the wire when explicitly given (the API defaults it itself).



192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/enconvert/v2.rb', line 192

def ingest(mode: nil, url: nil, urls: nil, max_pages: nil, max_depth: nil, same_domain_only: nil,
           include_patterns: nil, exclude_patterns: nil, respect_robots: nil, wait_for: nil,
           wait_timeout_ms: nil, chunk: nil, webhook_url: nil)
  effective_mode = mode || "urls"
  if effective_mode == "urls"
    raise Enconvert::Error, "ingest: mode 'urls' requires a non-empty 'urls' list" if urls.nil? || urls.empty?
    raise Enconvert::Error, "ingest: mode 'urls' does not accept 'url'" unless url.nil?
  else
    raise Enconvert::Error, "ingest: mode '#{effective_mode}' requires a seed 'url'" if url.nil?
    raise Enconvert::Error, "ingest: mode '#{effective_mode}' does not accept 'urls'" unless urls.nil?
  end

  body = {}
  body[:mode] = mode unless mode.nil?
  body[:url] = url unless url.nil?
  body[:urls] = urls unless urls.nil?
  body[:max_pages] = max_pages unless max_pages.nil?
  body[:max_depth] = max_depth unless max_depth.nil?
  body[:same_domain_only] = same_domain_only unless same_domain_only.nil?
  body[:include_patterns] = include_patterns unless include_patterns.nil?
  body[:exclude_patterns] = exclude_patterns unless exclude_patterns.nil?
  body[:respect_robots] = respect_robots unless respect_robots.nil?
  body[:wait_for] = wait_for unless wait_for.nil?
  body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
  if chunk
    c = {}
    c[:max_words] = chunk[:max_words] if chunk.key?(:max_words)
    c[:sentence_overlap] = chunk[:sentence_overlap] if chunk.key?(:sentence_overlap)
    body[:chunk] = c
  end
  body[:webhook_url] = webhook_url unless webhook_url.nil?
  to_ingest_job(post("/v2/ingest", body))
end

#ingest_files(files, chunk: nil, webhook_url: nil) ⇒ Object

Ingest one or more uploaded FILES into RAG-ready JSONL chunks — the file counterpart of ingest(), sharing the same job lifecycle (mode "files"). PDF, DOCX, PPTX, XLSX, CSV, HTML, EPUB, TXT/MD and legacy/ODF office are accepted. Always asynchronous; poll get_ingest_job or configure a webhook.

files accepts the same file-input union as convert_image (path string, IO-like/readable object, or { data:, filename: }); at least one file is required.



235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
 
# File 'lib/enconvert/v2.rb', line 235

def ingest_files(files, chunk: nil, webhook_url: nil)
  unless files.is_a?(Array) && !files.empty?
    raise Enconvert::Error, "ingest_files: provide at least one file"
  end

  fields = files.map do |file|
    part = to_file_part(file)
    { name: "files", value: part.bytes, filename: part.filename, content_type: part.content_type }
  end
  if chunk
    fields << { name: "max_words", value: chunk[:max_words] } if chunk.key?(:max_words)
    fields << { name: "sentence_overlap", value: chunk[:sentence_overlap] } if chunk.key?(:sentence_overlap)
  end
  fields << { name: "webhook_url", value: webhook_url } unless webhook_url.nil?

  body, boundary = build_multipart(fields)
  resp = @transport.request(
    :post, "/v2/ingest/files",
    headers: { "Content-Type" => "multipart/form-data; boundary=#{boundary}" },
    body: body
  )
  raise_for_status(resp)
  to_ingest_job(JSON.parse(resp.body))
end

#list_ingest_jobs(skip: nil, limit: nil) ⇒ Object

List ingest jobs, newest first.



261
262
263
264
265
266
267
268
269
270
 
# File 'lib/enconvert/v2.rb', line 261

def list_ingest_jobs(skip: nil, limit: nil)
  d = get("/v2/ingest#{list_query(skip, limit)}")
  jobs = d["jobs"].is_a?(Array) ? d["jobs"] : []
  IngestJobList.new(
    jobs: jobs.map { |j| to_ingest_job_summary(j) },
    skip: num(d["skip"]),
    limit: num(d["limit"], 20),
    has_more: d["has_more"] == true
  )
end

#list_watchers(skip: nil, limit: nil) ⇒ Object

List watchers, newest first.



326
327
328
329
330
331
332
333
334
335
 
# File 'lib/enconvert/v2.rb', line 326

def list_watchers(skip: nil, limit: nil)
  d = get("/v2/watch#{list_query(skip, limit)}")
  watchers = d["watchers"].is_a?(Array) ? d["watchers"] : []
  WatcherList.new(
    watchers: watchers.map { |w| to_watcher_summary(w) },
    skip: num(d["skip"]),
    limit: num(d["limit"], 20),
    has_more: d["has_more"] == true
  )
end

#lookup(query, **opts) ⇒ Object

Run a categorized web search. With perceive_top > 0, the top-N result URLs are auto-perceived (each consumes one perceive-quota unit) and carry their full PerceiveResult inline.

opts accepts: category, country, locale, time_filter, num_results, page, location, autocorrect, perceive_top.



134
135
136
137
138
139
140
141
142
143
144
145
146
 
# File 'lib/enconvert/v2.rb', line 134

def lookup(query, **opts)
  body = { query: query }
  body[:category] = opts[:category] if opts.key?(:category)
  body[:country] = opts[:country] if opts.key?(:country)
  body[:locale] = opts[:locale] if opts.key?(:locale)
  body[:time_filter] = opts[:time_filter] if opts.key?(:time_filter)
  body[:num_results] = opts[:num_results] if opts.key?(:num_results)
  body[:page] = opts[:page] if opts.key?(:page)
  body[:location] = opts[:location] if opts.key?(:location)
  body[:autocorrect] = opts[:autocorrect] if opts.key?(:autocorrect)
  body[:perceive_top] = opts[:perceive_top] if opts.key?(:perceive_top)
  to_lookup_result(post("/v2/lookup", body))
end

#perceive(url, **opts) ⇒ Object

Render one URL into the requested outputs (markdown, screenshots, PDF, links, structured data, ...). Synchronous: returns the completed operation with 15-minute signed artifact URLs.

opts accepts: outputs, extract, schema, wait_for, wait_timeout_ms, js_code, viewport, headers, cookies, auth, proxy_url, geolocation, action_chain, cache_mode, pdf_options, block_resources, respect_robots, mobile, only_main_content, direct_download.



42
43
44
45
46
47
48
49
50
 
# File 'lib/enconvert/v2.rb', line 42

def perceive(url, **opts)
  body = serialize_perceive_options(opts)
  body[:url] = url
  # direct_download is accepted by POST /v2/perceive only (the batch
  # endpoint rejects it with 422), so it stays out of the shared
  # options serializer.
  body[:direct_download] = opts[:direct_download] if opts.key?(:direct_download)
  to_perceive_result(post("/v2/perceive", body))
end

#perceive_batch(urls, output_mode: nil, **render_opts) ⇒ Object

Perceive up to 1000 URLs with one shared options block. Small batches run inline (completed result); larger ones return status "queued" — poll get_perceive_batch with the job_id.



92
93
94
95
96
 
# File 'lib/enconvert/v2.rb', line 92

def perceive_batch(urls, output_mode: nil, **render_opts)
  body = { urls: urls, options: serialize_perceive_options(render_opts) }
  body[:output_mode] = output_mode unless output_mode.nil?
  to_perceive_batch_result(post("/v2/perceive/batch", body))
end

#perceive_direct(url, **opts) ⇒ Object

Perceive with direct_download forced on: the response body IS the artifact bytes (content), with metadata in headers — no signed URL round-trip. Requires exactly one artifact-producing output ("structured" may ride along; it stays inline server-side and is not returned here). Accepts the same opts as perceive.



57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/enconvert/v2.rb', line 57

def perceive_direct(url, **opts)
  require_one_artifact_output("perceive_direct", opts)
  body = serialize_perceive_options(opts)
  body[:url] = url
  body[:direct_download] = true
  resp = @transport.request(
    :post, "/v2/perceive",
    headers: { "Content-Type" => "application/json" },
    body: JSON.generate(body)
  )
  raise_for_status(resp)
  to_perceive_direct_result(resp)
end

#retry_ingest_webhook(job_id) ⇒ Object

Re-deliver the completion webhook of a completed job (409 if the job is not completed, 400 if it has no webhook configured).



285
286
287
288
289
290
291
292
293
294
 
# File 'lib/enconvert/v2.rb', line 285

def retry_ingest_webhook(job_id)
  d = post("/v2/ingest/#{encode_path(job_id)}/retry-webhook", nil)
  WebhookRetryResult.new(
    job_id: str(d["job_id"]),
    delivered: d["delivered"] == true,
    attempts: num(d["attempts"]),
    status_code: opt_num(d["status_code"]),
    detail: str(d["detail"])
  )
end

#rotate_webhook_secret ⇒ Object

Rotate the webhook signing secret. Signatures made with the previous secret stop verifying immediately.



304
305
306
 
# File 'lib/enconvert/v2.rb', line 304

def rotate_webhook_secret
  to_webhook_secret(post("/v2/ingest/webhook-secret/rotate", nil))
end

#update_watcher(watcher_id, frequency_minutes: nil, diff_mode: nil, track_fields: nil, webhook_url: nil, notify_email: nil, status: nil) ⇒ Object

Update a watcher. At least one field is required. Set webhook_url to "" to clear the webhook; resuming a paused watcher re-checks the plan's watcher cap.

Raises:



357
358
359
360
361
362
363
364
365
366
367
368
369
 
# File 'lib/enconvert/v2.rb', line 357

def update_watcher(watcher_id, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
                    webhook_url: nil, notify_email: nil, status: nil)
  body = {}
  body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
  body[:diff_mode] = diff_mode unless diff_mode.nil?
  body[:track_fields] = track_fields unless track_fields.nil?
  body[:webhook_url] = webhook_url unless webhook_url.nil?
  body[:notify_email] = notify_email unless notify_email.nil?
  body[:status] = status unless status.nil?
  raise Enconvert::Error, "update_watcher: provide at least one field to update" if body.empty?

  to_watcher(patch("/v2/watch/#{encode_path(watcher_id)}", body))
end