Class: Gitlab::Triage::RestAPINetwork

Inherits:
Object
  • Object
show all
Includes:
Retryable
Defined in:
lib/gitlab/triage/rest_api_network.rb

Constant Summary collapse

MINIMUM_RATE_LIMIT = 
25
CACHE_URL_PATTERN = 
%r{/api/v[^/]+/(groups|projects)/(.+?)/(members)(?:\z|\?)}

Constants included from Retryable

Gitlab::Triage::Retryable::BACK_OFF_SECONDS, Gitlab::Triage::Retryable::MAX_RETRIES, Gitlab::Triage::Retryable::RETRY_WAIT_SECONDS

Instance Attribute Summary collapse

Attributes included from Retryable

#tries

Instance Method Summary collapse

Methods included from Retryable

#execute_with_retry, #maximum_retries_reached?, #puts_execute_with_retry_message

Constructor Details

#initialize(adapter) ⇒ RestAPINetwork

Returns a new instance of RestAPINetwork.



22
23
24
25
26
27
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 22

def initialize(adapter)
  @adapter = adapter
  @options = adapter.options
  @cache = {}
  @file_cache = {}
end

Instance Attribute Details

#adapter ⇒ Object (readonly)

Returns the value of attribute adapter.



20
21
22
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 20

def adapter
  @adapter
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



20
21
22
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 20

def options
  @options
end

Instance Method Details

#cache_dir ⇒ Object (private) 



108
109
110
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 108

def cache_dir
  options.cache_dir || ENV.fetch('GITLAB_TRIAGE_CACHE_DIR', nil)
end

#cache_dir_valid? ⇒ Boolean (private)

Returns:

  • (Boolean) 


121
122
123
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 121

def cache_dir_valid?
  cache_dir && File.directory?(cache_dir)
end

#cache_key_for(url) ⇒ Object (private) 



125
126
127
128
129
130
131
132
133
134
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 125

def cache_key_for(url)
  match = url.match(CACHE_URL_PATTERN)
  return nil unless match

  source_type = match[1]
  source_id   = URI.decode_www_form_component(match[2]).tr('/', '_')
  resource    = match[3]

  "#{source_type}_#{source_id}_#{resource}"
end

#delete_api(url) ⇒ Object 



93
94
95
96
97
98
99
100
101
102
103
104
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 93

def delete_api(url)
  response = execute_with_retry(
    exception_types: [Net::ReadTimeout, Errors::Network::InternalServerError],
    backoff_exceptions: Errors::Network::TooManyRequests, debug: options.debug) do
    puts Gitlab::Triage::UI.debug "delete_api: #{url}" if options.debug

    @adapter.delete(token, url)
  end

  rate_limit_debug(response) if options.debug
  rate_limit_wait(response)
end

#load_from_file_cache(url) ⇒ Object (private) 



112
113
114
115
116
117
118
119
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 112

def load_from_file_cache(url)
  return nil unless cache_dir_valid?

  cache_key = cache_key_for(url)
  return nil unless cache_key

  @file_cache[cache_key] || read_cache_file(cache_key)
end

#parse_cache_file(cache_key) ⇒ Object (private) 



145
146
147
148
149
150
151
152
153
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 145

def parse_cache_file(cache_key)
  file = File.join(cache_dir, "#{cache_key}.json")
  parsed = JSON.parse(File.read(file))
  return nil unless parsed.is_a?(Array) && parsed.all?(Hash)

  parsed.map(&:with_indifferent_access)
rescue JSON::ParserError, Errno::ENOENT, Errno::EACCES
  nil
end

#post_api(url, body) ⇒ Object 



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 71

def post_api(url, body)
  response = execute_with_retry(
    exception_types: [Net::ReadTimeout, Errors::Network::InternalServerError],
    backoff_exceptions: Errors::Network::TooManyRequests, debug: options.debug) do
    puts Gitlab::Triage::UI.debug "post_api: #{url}" if options.debug

    @adapter.post(token, url, body)
  end

  rate_limit_debug(response) if options.debug
  rate_limit_wait(response)

  results = response.delete(:results)

  case results
  when Hash
    results.with_indifferent_access
  else
    raise_unexpected_response(results)
  end
end

#query_api(url) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 33

def query_api(url)
  response = {}
  resources = []

  begin
    print '.'
    url = response.fetch(:next_page_url) { url }

    response = execute_with_retry(
      exception_types: [Net::ReadTimeout, Errors::Network::InternalServerError],
      backoff_exceptions: Errors::Network::TooManyRequests, debug: options.debug) do
      puts Gitlab::Triage::UI.debug "query_api: #{url}" if options.debug

      @adapter.get(token, url)
    end

    results = response.delete(:results)

    case results
    when Array
      resources.concat(results)
    when Hash
      if results['message']&.match?(/404 Group|Project Not Found/)
        raise_unexpected_response(results)
      else
        resources << results
      end
    else
      raise_unexpected_response(results)
    end

    rate_limit_debug(response) if options.debug
    rate_limit_wait(response)
  end while response.delete(:more_pages)

  resources.map!(&:with_indifferent_access)
end

#query_api_cached(url) ⇒ Object 



29
30
31
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 29

def query_api_cached(url)
  @cache[url] || @cache[url] = load_from_file_cache(url) || query_api(url)
end

#raise_unexpected_response(results) ⇒ Object (private)

Raises:



171
172
173
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 171

def raise_unexpected_response(results)
  raise Errors::Network::UnexpectedResponse, "Unexpected response: #{results.inspect}"
end

#rate_limit_debug(response) ⇒ Object (private) 



159
160
161
162
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 159

def rate_limit_debug(response)
  rate_limit_infos = "Rate limit remaining: #{response[:ratelimit_remaining]} (reset at #{response[:ratelimit_reset_at]})"
  puts Gitlab::Triage::UI.debug "rate_limit_infos: #{rate_limit_infos}"
end

#rate_limit_wait(response) ⇒ Object (private) 



164
165
166
167
168
169
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 164

def rate_limit_wait(response)
  return unless response.delete(:ratelimit_remaining) < MINIMUM_RATE_LIMIT

  puts Gitlab::Triage::UI.debug "Rate limit almost exceeded, sleeping for #{response[:ratelimit_reset_at] - Time.now} seconds" if options.debug
  sleep(1) until Time.now >= response[:ratelimit_reset_at]
end

#read_cache_file(cache_key) ⇒ Object (private) 



136
137
138
139
140
141
142
143
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 136

def read_cache_file(cache_key)
  data = parse_cache_file(cache_key)
  return nil unless data

  @file_cache[cache_key] = data
  puts Gitlab::Triage::UI.debug "cache_dir: loaded #{data.size} records" if options.debug
  data
end

#token ⇒ Object (private) 



155
156
157
 
# File 'lib/gitlab/triage/rest_api_network.rb', line 155

def token
  options.token
end