Class: Enconvert::V2

Inherits:
Object
  • Object
show all
Includes:
Internal
Defined in:
lib/enconvert/v2.rb,
lib/enconvert/v2_results.rb

Defined Under Namespace

Modules: Enums Classes: DiscoverResult, DistillItem, DistillResult, IngestJob, IngestJobList, IngestJobSummary, LookupItem, LookupResult, PerceiveBatchResult, PerceiveResult, V2OutputArtifact, V2Tokens, Watcher, WatcherList, WatcherSnapshot, WatcherSnapshotList, WatcherSummary, WebhookRetryResult, WebhookSecret

Instance Method Summary collapse

Methods included from Internal

build_multipart, extract_error_message, new_job_id, raise_for_status, serialize_pdf_options, to_file_part

Constructor Details

#initialize(transport) ⇒ V2

Returns a new instance of V2.



26
27
28
 
# File 'lib/enconvert/v2.rb', line 26

def initialize(transport)
  @transport = transport
end

Instance Method Details

#cancel_ingest_job(job_id) ⇒ Object

Cancel a queued/processing ingest job. Idempotent: canceling an already-terminal job returns it unchanged.



244
245
246
 
# File 'lib/enconvert/v2.rb', line 244

def cancel_ingest_job(job_id)
  to_ingest_job(delete_request("/v2/ingest/#{encode_path(job_id)}"))
end

#create_watcher(url, frequency_minutes: nil, diff_mode: nil, track_fields: nil, webhook_url: nil, notify_email: nil) ⇒ Object

Create a watcher that re-renders url on a fixed cadence (hourly floor) and notifies on changes via email and/or webhook.



279
280
281
282
283
284
285
286
287
288
 
# File 'lib/enconvert/v2.rb', line 279

def create_watcher(url, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
                    webhook_url: nil, notify_email: nil)
  body = { url: url }
  body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
  body[:diff_mode] = diff_mode unless diff_mode.nil?
  body[:track_fields] = track_fields unless track_fields.nil?
  body[:webhook_url] = webhook_url unless webhook_url.nil?
  body[:notify_email] = notify_email unless notify_email.nil?
  to_watcher(post("/v2/watch", body))
end

#delete_watcher(watcher_id) ⇒ Object

Soft-delete a watcher (idempotent). Returns the tombstoned watcher with status "deleted".



338
339
340
 
# File 'lib/enconvert/v2.rb', line 338

def delete_watcher(watcher_id)
  to_watcher(delete_request("/v2/watch/#{encode_path(watcher_id)}"))
end

#discover(url, **opts) ⇒ Object

List a site's URLs via sitemap, HTTP crawl, or both. No browser rendering — fast and does not consume perceive quota.

opts accepts: mode, max_urls, max_depth, include_patterns, exclude_patterns, same_domain_only, respect_robots.



77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/enconvert/v2.rb', line 77

def discover(url, **opts)
  body = { url: url }
  body[:mode] = opts[:mode] if opts.key?(:mode)
  body[:max_urls] = opts[:max_urls] if opts.key?(:max_urls)
  body[:max_depth] = opts[:max_depth] if opts.key?(:max_depth)
  body[:include_patterns] = opts[:include_patterns] if opts.key?(:include_patterns)
  body[:exclude_patterns] = opts[:exclude_patterns] if opts.key?(:exclude_patterns)
  body[:same_domain_only] = opts[:same_domain_only] if opts.key?(:same_domain_only)
  body[:respect_robots] = opts[:respect_robots] if opts.key?(:respect_robots)
  to_discover_result(post("/v2/discover", body))
end

#distill(urls: nil, discover_from: nil, schema: nil, css_schema: nil, wait_for: nil, wait_timeout_ms: nil, headers: nil, cookies: nil, respect_robots: nil) ⇒ Object

Extract structured data matching schema from explicit URLs or from a discovered site. An optional css_schema answers fields for free; anything it misses escalates to the LLM tier (plan-gated).

Provide exactly one of urls (non-empty Array) or discover_from (Hash with :url, :mode, :max_pages). schema is required.

Raises:



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/enconvert/v2.rb', line 123

def distill(urls: nil, discover_from: nil, schema: nil, css_schema: nil, wait_for: nil,
            wait_timeout_ms: nil, headers: nil, cookies: nil, respect_robots: nil)
  has_urls = urls.is_a?(Array) && !urls.empty?
  has_discover = !discover_from.nil?
  raise Enconvert::Error, "distill: provide exactly one of 'urls' or 'discover_from'" if has_urls == has_discover
  raise Enconvert::Error, "distill: 'schema' is required and must be a Hash" unless schema.is_a?(Hash)

  body = { schema: schema }
  body[:urls] = urls if has_urls
  if discover_from
    df = { url: discover_from[:url] }
    df[:mode] = discover_from[:mode] if discover_from.key?(:mode)
    df[:max_pages] = discover_from[:max_pages] if discover_from.key?(:max_pages)
    body[:discover_from] = df
  end
  body[:css_schema] = serialize_css_schema(css_schema) unless css_schema.nil?
  body[:wait_for] = wait_for unless wait_for.nil?
  body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
  body[:headers] = headers unless headers.nil?
  body[:cookies] = cookies unless cookies.nil?
  body[:respect_robots] = respect_robots unless respect_robots.nil?
  to_distill_result(post("/v2/distill", body))
end

#get_ingest_job(job_id) ⇒ Object

Get one ingest job by id (ing_...).



238
239
240
 
# File 'lib/enconvert/v2.rb', line 238

def get_ingest_job(job_id)
  to_ingest_job(get("/v2/ingest/#{encode_path(job_id)}"))
end

#get_perceive_batch(job_id) ⇒ Object

Poll a perceive batch by job_id. Items fill in as URLs complete.



64
65
66
 
# File 'lib/enconvert/v2.rb', line 64

def get_perceive_batch(job_id)
  to_perceive_batch_result(get("/v2/perceive/batch/#{encode_path(job_id)}"))
end

#get_perceive_operation(operation_id) ⇒ Object

Re-fetch a perceive operation by id (per_...). Artifact URLs are freshly re-signed on every call.



50
51
52
 
# File 'lib/enconvert/v2.rb', line 50

def get_perceive_operation(operation_id)
  to_perceive_result(get("/v2/perceive/#{encode_path(operation_id)}"))
end

#get_watcher(watcher_id) ⇒ Object

Get one watcher by id (wat_...). Deleted watchers read as 404.



303
304
305
 
# File 'lib/enconvert/v2.rb', line 303

def get_watcher(watcher_id)
  to_watcher(get("/v2/watch/#{encode_path(watcher_id)}"))
end

#get_watcher_snapshots(watcher_id, limit: nil) ⇒ Object

Page through a watcher's check history, newest first.



308
309
310
311
312
313
314
315
316
317
 
# File 'lib/enconvert/v2.rb', line 308

def get_watcher_snapshots(watcher_id, limit: nil)
  query = limit.nil? ? "" : "?limit=#{limit}"
  d = get("/v2/watch/#{encode_path(watcher_id)}/snapshots#{query}")
  snapshots = d["snapshots"].is_a?(Array) ? d["snapshots"] : []
  WatcherSnapshotList.new(
    watcher_id: str(d["watcher_id"]),
    snapshots: snapshots.map { |s| to_watcher_snapshot(s) },
    limit: num(d["limit"], 20)
  )
end

#get_webhook_secret ⇒ Object

Get (creating on first call) the project's webhook signing secret and the header/scheme details needed to verify deliveries.



263
264
265
 
# File 'lib/enconvert/v2.rb', line 263

def get_webhook_secret
  to_webhook_secret(get("/v2/ingest/webhook-secret"))
end

#ingest(mode: nil, url: nil, urls: nil, max_pages: nil, max_depth: nil, same_domain_only: nil, include_patterns: nil, exclude_patterns: nil, respect_robots: nil, wait_for: nil, wait_timeout_ms: nil, chunk: nil, webhook_url: nil) ⇒ Object

Start an ingest job: turn explicit URLs or a discovered site into chunked, RAG-ready JSONL. Always asynchronous — returns the queued job; poll get_ingest_job or configure webhook_url for completion.

mode defaults to "urls" for validation purposes but is only sent on the wire when explicitly given (the API defaults it itself).



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/enconvert/v2.rb', line 157

def ingest(mode: nil, url: nil, urls: nil, max_pages: nil, max_depth: nil, same_domain_only: nil,
           include_patterns: nil, exclude_patterns: nil, respect_robots: nil, wait_for: nil,
           wait_timeout_ms: nil, chunk: nil, webhook_url: nil)
  effective_mode = mode || "urls"
  if effective_mode == "urls"
    raise Enconvert::Error, "ingest: mode 'urls' requires a non-empty 'urls' list" if urls.nil? || urls.empty?
    raise Enconvert::Error, "ingest: mode 'urls' does not accept 'url'" unless url.nil?
  else
    raise Enconvert::Error, "ingest: mode '#{effective_mode}' requires a seed 'url'" if url.nil?
    raise Enconvert::Error, "ingest: mode '#{effective_mode}' does not accept 'urls'" unless urls.nil?
  end

  body = {}
  body[:mode] = mode unless mode.nil?
  body[:url] = url unless url.nil?
  body[:urls] = urls unless urls.nil?
  body[:max_pages] = max_pages unless max_pages.nil?
  body[:max_depth] = max_depth unless max_depth.nil?
  body[:same_domain_only] = same_domain_only unless same_domain_only.nil?
  body[:include_patterns] = include_patterns unless include_patterns.nil?
  body[:exclude_patterns] = exclude_patterns unless exclude_patterns.nil?
  body[:respect_robots] = respect_robots unless respect_robots.nil?
  body[:wait_for] = wait_for unless wait_for.nil?
  body[:wait_timeout_ms] = wait_timeout_ms unless wait_timeout_ms.nil?
  if chunk
    c = {}
    c[:max_words] = chunk[:max_words] if chunk.key?(:max_words)
    c[:sentence_overlap] = chunk[:sentence_overlap] if chunk.key?(:sentence_overlap)
    body[:chunk] = c
  end
  body[:webhook_url] = webhook_url unless webhook_url.nil?
  to_ingest_job(post("/v2/ingest", body))
end

#ingest_files(files, chunk: nil, webhook_url: nil) ⇒ Object

Ingest one or more uploaded FILES into RAG-ready JSONL chunks — the file counterpart of ingest(), sharing the same job lifecycle (mode "files"). PDF, DOCX, PPTX, XLSX, CSV, HTML, EPUB, TXT/MD and legacy/ODF office are accepted. Always asynchronous; poll get_ingest_job or configure a webhook.

files accepts the same file-input union as convert_image (path string, IO-like/readable object, or { data:, filename: }); at least one file is required.



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
 
# File 'lib/enconvert/v2.rb', line 200

def ingest_files(files, chunk: nil, webhook_url: nil)
  unless files.is_a?(Array) && !files.empty?
    raise Enconvert::Error, "ingest_files: provide at least one file"
  end

  fields = files.map do |file|
    part = to_file_part(file)
    { name: "files", value: part.bytes, filename: part.filename, content_type: part.content_type }
  end
  if chunk
    fields << { name: "max_words", value: chunk[:max_words] } if chunk.key?(:max_words)
    fields << { name: "sentence_overlap", value: chunk[:sentence_overlap] } if chunk.key?(:sentence_overlap)
  end
  fields << { name: "webhook_url", value: webhook_url } unless webhook_url.nil?

  body, boundary = build_multipart(fields)
  resp = @transport.request(
    :post, "/v2/ingest/files",
    headers: { "Content-Type" => "multipart/form-data; boundary=#{boundary}" },
    body: body
  )
  raise_for_status(resp)
  to_ingest_job(JSON.parse(resp.body))
end

#list_ingest_jobs(skip: nil, limit: nil) ⇒ Object

List ingest jobs, newest first.



226
227
228
229
230
231
232
233
234
235
 
# File 'lib/enconvert/v2.rb', line 226

def list_ingest_jobs(skip: nil, limit: nil)
  d = get("/v2/ingest#{list_query(skip, limit)}")
  jobs = d["jobs"].is_a?(Array) ? d["jobs"] : []
  IngestJobList.new(
    jobs: jobs.map { |j| to_ingest_job_summary(j) },
    skip: num(d["skip"]),
    limit: num(d["limit"], 20),
    has_more: d["has_more"] == true
  )
end

#list_watchers(skip: nil, limit: nil) ⇒ Object

List watchers, newest first.



291
292
293
294
295
296
297
298
299
300
 
# File 'lib/enconvert/v2.rb', line 291

def list_watchers(skip: nil, limit: nil)
  d = get("/v2/watch#{list_query(skip, limit)}")
  watchers = d["watchers"].is_a?(Array) ? d["watchers"] : []
  WatcherList.new(
    watchers: watchers.map { |w| to_watcher_summary(w) },
    skip: num(d["skip"]),
    limit: num(d["limit"], 20),
    has_more: d["has_more"] == true
  )
end

#lookup(query, **opts) ⇒ Object

Run a categorized web search. With perceive_top > 0, the top-N result URLs are auto-perceived (each consumes one perceive-quota unit) and carry their full PerceiveResult inline.

opts accepts: category, country, locale, time_filter, num_results, page, location, autocorrect, perceive_top.



99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/enconvert/v2.rb', line 99

def lookup(query, **opts)
  body = { query: query }
  body[:category] = opts[:category] if opts.key?(:category)
  body[:country] = opts[:country] if opts.key?(:country)
  body[:locale] = opts[:locale] if opts.key?(:locale)
  body[:time_filter] = opts[:time_filter] if opts.key?(:time_filter)
  body[:num_results] = opts[:num_results] if opts.key?(:num_results)
  body[:page] = opts[:page] if opts.key?(:page)
  body[:location] = opts[:location] if opts.key?(:location)
  body[:autocorrect] = opts[:autocorrect] if opts.key?(:autocorrect)
  body[:perceive_top] = opts[:perceive_top] if opts.key?(:perceive_top)
  to_lookup_result(post("/v2/lookup", body))
end

#perceive(url, **opts) ⇒ Object

Render one URL into the requested outputs (markdown, screenshots, PDF, links, structured data, ...). Synchronous: returns the completed operation with 15-minute signed artifact URLs.

opts accepts: outputs, extract, schema, wait_for, wait_timeout_ms, js_code, viewport, headers, cookies, auth, proxy_url, geolocation, action_chain, cache_mode, pdf_options, block_resources, respect_robots, mobile.



42
43
44
45
46
 
# File 'lib/enconvert/v2.rb', line 42

def perceive(url, **opts)
  body = serialize_perceive_options(opts)
  body[:url] = url
  to_perceive_result(post("/v2/perceive", body))
end

#perceive_batch(urls, output_mode: nil, **render_opts) ⇒ Object

Perceive up to 1000 URLs with one shared options block. Small batches run inline (completed result); larger ones return status "queued" — poll get_perceive_batch with the job_id.



57
58
59
60
61
 
# File 'lib/enconvert/v2.rb', line 57

def perceive_batch(urls, output_mode: nil, **render_opts)
  body = { urls: urls, options: serialize_perceive_options(render_opts) }
  body[:output_mode] = output_mode unless output_mode.nil?
  to_perceive_batch_result(post("/v2/perceive/batch", body))
end

#retry_ingest_webhook(job_id) ⇒ Object

Re-deliver the completion webhook of a completed job (409 if the job is not completed, 400 if it has no webhook configured).



250
251
252
253
254
255
256
257
258
259
 
# File 'lib/enconvert/v2.rb', line 250

def retry_ingest_webhook(job_id)
  d = post("/v2/ingest/#{encode_path(job_id)}/retry-webhook", nil)
  WebhookRetryResult.new(
    job_id: str(d["job_id"]),
    delivered: d["delivered"] == true,
    attempts: num(d["attempts"]),
    status_code: opt_num(d["status_code"]),
    detail: str(d["detail"])
  )
end

#rotate_webhook_secret ⇒ Object

Rotate the webhook signing secret. Signatures made with the previous secret stop verifying immediately.



269
270
271
 
# File 'lib/enconvert/v2.rb', line 269

def rotate_webhook_secret
  to_webhook_secret(post("/v2/ingest/webhook-secret/rotate", nil))
end

#update_watcher(watcher_id, frequency_minutes: nil, diff_mode: nil, track_fields: nil, webhook_url: nil, notify_email: nil, status: nil) ⇒ Object

Update a watcher. At least one field is required. Set webhook_url to "" to clear the webhook; resuming a paused watcher re-checks the plan's watcher cap.

Raises:



322
323
324
325
326
327
328
329
330
331
332
333
334
 
# File 'lib/enconvert/v2.rb', line 322

def update_watcher(watcher_id, frequency_minutes: nil, diff_mode: nil, track_fields: nil,
                    webhook_url: nil, notify_email: nil, status: nil)
  body = {}
  body[:frequency_minutes] = frequency_minutes unless frequency_minutes.nil?
  body[:diff_mode] = diff_mode unless diff_mode.nil?
  body[:track_fields] = track_fields unless track_fields.nil?
  body[:webhook_url] = webhook_url unless webhook_url.nil?
  body[:notify_email] = notify_email unless notify_email.nil?
  body[:status] = status unless status.nil?
  raise Enconvert::Error, "update_watcher: provide at least one field to update" if body.empty?

  to_watcher(patch("/v2/watch/#{encode_path(watcher_id)}", body))
end