Module: HttpMimic

Extended by:
ModuleMethods
Defined in:
lib/http_mimic.rb,
lib/http_mimic/waf.rb,
lib/http_mimic/client.rb,
lib/http_mimic/cookies.rb,
lib/http_mimic/headers.rb,
lib/http_mimic/obscura.rb,
lib/http_mimic/request.rb,
lib/http_mimic/version.rb,
lib/http_mimic/response.rb,
lib/http_mimic/downloader.rb,
lib/http_mimic/exceptions.rb,
lib/http_mimic/js_runtime.rb,
lib/http_mimic/cookie_store.rb,
lib/http_mimic/spa_detector.rb,
lib/http_mimic/waf/detector.rb,
lib/http_mimic/configuration.rb,
lib/http_mimic/module_methods.rb,
lib/http_mimic/command_builder.rb,
lib/http_mimic/response_parser.rb,
lib/http_mimic/waf/akamai_solver.rb,
lib/http_mimic/waf/google_solver.rb

Defined Under Namespace

Modules: InstanceMethods, ModuleMethods, Obscura, SpaDetector, Waf Classes: BinaryNotFoundError, Client, CommandBuilder, CommandError, Configuration, ConnectionError, CookieStore, Cookies, Downloader, Error, Headers, JSError, JSRuntime, JSTimeoutError, Request, Response, ResponseParseError, ResponseParser, SSLError, TimeoutError

Constant Summary collapse

VERSION = 
"0.5.1"

Class Method Summary collapse

Methods included from ModuleMethods

auto_fallback, auto_render_spa, base_uri, cookies, default_options, default_params, default_timeout, delete, get, head, headers, impersonate, mode, options, patch, persist_cookies, post, proxy, put, render, request, retry_statuses

Class Method Details

.clear_cookies!(host = nil) ⇒ Object 



95
96
97
 
# File 'lib/http_mimic.rb', line 95

def clear_cookies!(host = nil)
  CookieStore.clear(host)
end

.configuration ⇒ Object 



29
30
31
 
# File 'lib/http_mimic.rb', line 29

def configuration
  @configuration ||= Configuration.new
end

.configure {|configuration| ... } ⇒ Object

Yields:



33
34
35
 
# File 'lib/http_mimic.rb', line 33

def configure
  yield(configuration) if block_given?
end

.detect_waf(response) ⇒ Object

Automatically detect WAF type from response



77
78
79
 
# File 'lib/http_mimic.rb', line 77

def detect_waf(response)
  Waf::Detector.detect(response)
end

.download_driver!(version: nil, force: false) ⇒ Object

Manually download and install curl-impersonate binary driver



42
43
44
 
# File 'lib/http_mimic.rb', line 42

def download_driver!(version: nil, force: false)
  Downloader.download!(version: version, force: force)
end

.download_obscura!(version: nil, force: false) ⇒ Object

Obscura headless SPA engine helpers



100
101
102
 
# File 'lib/http_mimic.rb', line 100

def download_obscura!(version: nil, force: false)
  Downloader.download_obscura!(version: version, force: force)
end

.download_qjs!(version: nil, force: false) ⇒ Object

Manually download and install QuickJS binary driver



52
53
54
 
# File 'lib/http_mimic.rb', line 52

def download_qjs!(version: nil, force: false)
  Downloader.download_qjs!(version: version, force: force)
end

.driver_installed?(version: nil) ⇒ Boolean

Check if curl-impersonate driver is installed locally

Returns:

  • (Boolean) 


47
48
49
 
# File 'lib/http_mimic.rb', line 47

def driver_installed?(version: nil)
  Downloader.installed?(version: version)
end

.eval_js(code, options = {}) ⇒ Object

Evaluate JavaScript code using QuickJS



67
68
69
 
# File 'lib/http_mimic.rb', line 67

def eval_js(code, options = {})
  JSRuntime.eval(code, timeout: options[:timeout], install_dir: options[:install_dir], auto_download: options[:auto_download])
end

.eval_js_json(code, options = {}) ⇒ Object

Evaluate JavaScript and auto-parse JSON result



72
73
74
 
# File 'lib/http_mimic.rb', line 72

def eval_js_json(code, options = {})
  JSRuntime.eval_json(code, timeout: options[:timeout], install_dir: options[:install_dir], auto_download: options[:auto_download])
end

.included(base) ⇒ Object 



123
124
125
126
 
# File 'lib/http_mimic.rb', line 123

def included(base)
  base.extend(ModuleMethods)
  base.send(:include, InstanceMethods)
end

.load_cookies(host, max_age: nil) ⇒ Object

Persistent Host CookieStore helpers



87
88
89
 
# File 'lib/http_mimic.rb', line 87

def load_cookies(host, max_age: nil)
  CookieStore.load(host, max_age: max_age)
end

.obscura_installed?(version: nil) ⇒ Boolean

Returns:

  • (Boolean) 


104
105
106
 
# File 'lib/http_mimic.rb', line 104

def obscura_installed?(version: nil)
  Downloader.obscura_installed?(version: version)
end

.obscura_path ⇒ Object 



108
109
110
 
# File 'lib/http_mimic.rb', line 108

def obscura_path
  Downloader.obscura_path
end

.qjs_installed?(version: nil) ⇒ Boolean

Check if QuickJS driver is installed locally

Returns:

  • (Boolean) 


57
58
59
 
# File 'lib/http_mimic.rb', line 57

def qjs_installed?(version: nil)
  Downloader.qjs_installed?(version: version)
end

.qjs_path ⇒ Object

Return local path to QuickJS binary



62
63
64
 
# File 'lib/http_mimic.rb', line 62

def qjs_path
  Downloader.qjs_path
end

.render(url, options = {}) ⇒ Object Also known as: spa

Render SPA page using Obscura headless engine



113
114
115
 
# File 'lib/http_mimic.rb', line 113

def render(url, options = {})
  Obscura.render(url, options)
end

.reset_configuration! ⇒ Object 



37
38
39
 
# File 'lib/http_mimic.rb', line 37

def reset_configuration!
  @configuration = Configuration.new
end

.save_cookies(host, cookies, ttl: nil) ⇒ Object 



91
92
93
 
# File 'lib/http_mimic.rb', line 91

def save_cookies(host, cookies, ttl: nil)
  CookieStore.save(host, cookies, ttl: ttl || CookieStore::DEFAULT_TTL)
end

.solve_waf(url, response, options = {}) ⇒ Object

Attempt to solve WAF challenge for a blocked response



82
83
84
 
# File 'lib/http_mimic.rb', line 82

def solve_waf(url, response, options = {})
  Waf.solve(url, response, options)
end

.spa?(response) ⇒ Boolean

Check if a response represents an unhydrated SPA shell

Returns:

  • (Boolean) 


119
120
121
 
# File 'lib/http_mimic.rb', line 119

def spa?(response)
  SpaDetector.spa?(response)
end