Class: HttpMimic::Request

Inherits:
Object
  • Object
show all
Defined in:
lib/http_mimic/request.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(method, url, options = {}, config = nil) ⇒ Request

Returns a new instance of Request.



9
10
11
12
13
14
 
# File 'lib/http_mimic/request.rb', line 9

def initialize(method, url, options = {}, config = nil)
  @method  = method
  @url     = url
  @options = options.dup
  @config  = config || HttpMimic.configuration
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



7
8
9
 
# File 'lib/http_mimic/request.rb', line 7

def config
  @config
end

#method ⇒ Object (readonly)

Returns the value of attribute method.



7
8
9
 
# File 'lib/http_mimic/request.rb', line 7

def method
  @method
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



7
8
9
 
# File 'lib/http_mimic/request.rb', line 7

def options
  @options
end

#url ⇒ Object (readonly)

Returns the value of attribute url.



7
8
9
 
# File 'lib/http_mimic/request.rb', line 7

def url
  @url
end

Instance Method Details

#perform ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/http_mimic/request.rb', line 16

def perform
  mode = (options[:mode] || config.mode || :auto).to_sym
  auto_fallback = options.fetch(:auto_fallback, config.auto_fallback)
  retry_statuses = options[:retry_statuses] || config.retry_statuses || [403, 429, 503]

  # Persistent CookieStore integration
  should_persist_cookie = options.fetch(:persist_cookies, config.persist_cookies)
  host = CookieStore.extract_host(url)

  if should_persist_cookie && host
    stored_cookies = CookieStore.load(host)
    if stored_cookies && !stored_cookies.empty?
      log_debug("[HttpMimic::CookieStore] Loaded #{stored_cookies.size} persistent cookies for #{host}")
      user_cookies = options[:cookies] ? (options[:cookies].is_a?(Hash) ? options[:cookies] : options[:cookies].to_h) : {}
      options[:cookies] = stored_cookies.merge(user_cookies)
    end
  end

  attempts = []
  profiles_to_try = determine_profiles(mode, auto_fallback)
  waf_solve_attempted = false

  response = nil
  final_status = nil
  final_stderr = nil
  final_command = nil

  profiles_to_try.each_with_index do |profile, index|
    current_opts = options.merge(profile: profile)

    builder = CommandBuilder.new(method, url, current_opts, config)
    binary, args, stdin_data, final_url = builder.build
    full_command = [binary] + args

    log_debug("Executing HttpMimic command (attempt #{index + 1}, profile: #{profile}): #{full_command.join(' ')}")
    log_debug("Stdin data: #{stdin_data}") if stdin_data

    stdout, stderr, status = execute_open3(full_command, stdin_data)

    log_debug("Curl exit status: #{status.exitstatus}")
    log_debug("Curl stderr: #{stderr}") unless stderr.empty?

    response = ResponseParser.new(
      stdout,
      exit_status: status,
      stderr: stderr,
      command: full_command,
      request_url: final_url
    ).parse

    response.mode_used = profile
    response.fallback_triggered = (index > 0)

    attempts << {
      attempt: index + 1,
      profile: profile,
      command: full_command,
      code: response.code,
      exit_code: status.exitstatus,
      success: response.success?
    }
    response.attempts = attempts

    final_status = status
    final_stderr = stderr
    final_command = full_command

    # Forward any received cookies to subsequent attempts
    if response.cookies && !response.cookies.empty?
      existing_cookies = options[:cookies] ? (options[:cookies].is_a?(Hash) ? options[:cookies] : options[:cookies].to_h) : {}
      options[:cookies] = response.cookies.to_h.merge(existing_cookies)
    end

    auto_solve_waf = options.fetch(:solve_waf, config.auto_solve_waf)
    is_blocked = (status.exitstatus != 0) || retry_statuses.include?(response.code) || Waf::Detector.challenge_page?(response)

    # Only attempt WAF resolution once per request to avoid unnecessary latency on subsequent fallbacks
    if is_blocked && auto_solve_waf && !waf_solve_attempted && method.to_s.upcase == 'GET'
      waf_type = Waf::Detector.detect(response)
      if waf_type
        waf_solve_attempted = true
        log_debug("[HttpMimic] Detected #{waf_type.to_s.capitalize} WAF challenge. Attempting to solve with QuickJS...")
        solved_resp = Waf.solve(url, response, current_opts)
        if solved_resp
          response = solved_resp
          is_blocked = (response.code != 0 && retry_statuses.include?(response.code))
          if response.cookies && !response.cookies.empty?
            options[:cookies] = (options[:cookies] || {}).merge(response.cookies.to_h)
          end
        end
      end
    end

    if !is_blocked || (index == profiles_to_try.size - 1)
      break
    end

    log_debug("[HttpMimic] Attempt #{index + 1} with #{profile} resulted in status #{response.code}. Triggering smart fallback to next profile...")
  end

  # Persist cookies back to store if enabled
  if should_persist_cookie && host && response && response.cookies && !response.cookies.empty?
    CookieStore.save(host, response.cookies)
    log_debug("[HttpMimic::CookieStore] Saved #{response.cookies.size} cookies for #{host}")
  end

  handle_errors(final_status, final_stderr, final_command, response)
  response
end