Class: Downloader

Inherits:
Object
  • Object
show all
Defined in:
lib/jirametrics/downloader.rb

Direct Known Subclasses

DownloaderForCloud, DownloaderForDataCenter

Constant Summary collapse

CURRENT_METADATA_VERSION = 
5

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(download_config:, file_system:, jira_gateway:) ⇒ Downloader

Returns a new instance of Downloader.



45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/jirametrics/downloader.rb', line 45

def initialize download_config:, file_system:, jira_gateway:
  @metadata = {}
  @download_config = download_config
  @target_path = @download_config.project_config.target_path
  @file_system = file_system
  @jira_gateway = jira_gateway
  @board_id_to_filter_id = {}

  @issue_keys_downloaded_in_current_run = []
  @issue_keys_pending_download = []
end

Instance Attribute Details

#board_id_to_filter_id ⇒ Object (readonly)

For testing only



34
35
36
 
# File 'lib/jirametrics/downloader.rb', line 34

def board_id_to_filter_id
  @board_id_to_filter_id
end

#file_system ⇒ Object (readonly)

Returns the value of attribute file_system.



31
32
33
 
# File 'lib/jirametrics/downloader.rb', line 31

def file_system
  @file_system
end

#metadata ⇒ Object

Returns the value of attribute metadata.



30
31
32
 
# File 'lib/jirametrics/downloader.rb', line 30

def metadata
  @metadata
end

#start_date_in_query ⇒ Object (readonly)

For testing only



34
35
36
 
# File 'lib/jirametrics/downloader.rb', line 34

def start_date_in_query
  @start_date_in_query
end

Class Method Details

.create(download_config:, file_system:, jira_gateway:) ⇒ Object 



36
37
38
39
40
41
42
43
 
# File 'lib/jirametrics/downloader.rb', line 36

def self.create download_config:, file_system:, jira_gateway:
  is_cloud = jira_gateway.settings['jira_cloud'] || jira_gateway.cloud?
  (is_cloud ? DownloaderForCloud : DownloaderForDataCenter).new(
    download_config: download_config,
    file_system: file_system,
    jira_gateway: jira_gateway
  )
end

Instance Method Details

#download_board_configuration(board_id:) ⇒ Object 



156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/jirametrics/downloader.rb', line 156

def download_board_configuration board_id:
  log "  Downloading board configuration for board #{board_id}", both: true
  json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"

  @file_system.save_json(
    json: json,
    filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
  )

  # We have a reported bug that blew up on this line. Moved it after the save so we can
  # actually look at the returned json.
  @board_id_to_filter_id[board_id] = json['filter']['id'].to_i

  download_sprints board_id: board_id if json['type'] == 'scrum'
  # TODO: Should be passing actual statuses, not empty list
  Board.new raw: json, possible_statuses: StatusCollection.new
end

#download_github_prs ⇒ Object 



305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
 
# File 'lib/jirametrics/downloader.rb', line 305

def download_github_prs
  project_keys = extract_project_keys_from_downloaded_issues
  if project_keys.empty?
    log '  No project keys found in downloaded issues, skipping GitHub PR download', both: true
    return
  end

  prs = @download_config.github_repos.flat_map do |repo|
    GithubGateway.new(
      repo: repo,
      project_keys: project_keys,
      file_system: @file_system
    ).fetch_pull_requests(since: @download_date_range&.begin)
  end

  @file_system.save_json(
    json: prs,
    filename: File.join(@target_path, "#{file_prefix}_github_prs.json")
  )
end

#download_sprints(board_id:) ⇒ Object 



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/jirametrics/downloader.rb', line 174

def download_sprints board_id:
  log "  Downloading sprints for board #{board_id}", both: true
  max_results = 100
  start_at = 0
  is_last = false

  while is_last == false
    json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
      "maxResults=#{max_results}&startAt=#{start_at}"

    @file_system.save_json(
      json: json,
      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json")
    )
    is_last = json['isLast']
    max_results = json['maxResults']
    if json['values']
      start_at += json['values'].size
    else
      log "  No sprints found for board #{board_id}"
    end
  end
end

#download_statuses ⇒ Object 



108
109
110
111
112
113
114
115
116
 
# File 'lib/jirametrics/downloader.rb', line 108

def download_statuses
  log '  Downloading all statuses', both: true
  json = @jira_gateway.call_url relative_url: '/rest/api/2/status'

  @file_system.save_json(
    json: json,
    filename: File.join(@target_path, "#{file_prefix}_statuses.json")
  )
end

#download_users ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
 
# File 'lib/jirametrics/downloader.rb', line 118

def download_users
  return unless @jira_gateway.cloud?

  log '  Downloading all users', both: true
  json = @jira_gateway.call_url relative_url: '/rest/api/2/users'

  @file_system.save_json(
    json: json,
    filename: File.join(@target_path, "#{file_prefix}_users.json")
  )
end

#extract_project_keys_from_downloaded_issues ⇒ Object 



326
327
328
329
330
331
332
333
334
335
 
# File 'lib/jirametrics/downloader.rb', line 326

def extract_project_keys_from_downloaded_issues
  path = File.join(@target_path, "#{file_prefix}_issues")
  return [] unless @file_system.dir_exist?(path)

  keys = []
  @file_system.foreach(path) do |filename|
    keys << filename.split('-').first if filename =~ /^[A-Z]+-\d+-\d+\.json$/
  end
  keys.uniq
end

#file_prefix ⇒ Object 



337
338
339
 
# File 'lib/jirametrics/downloader.rb', line 337

def file_prefix
  @download_config.project_config.get_file_prefix
end

#find_board_ids ⇒ Object 



87
88
89
90
91
92
 
# File 'lib/jirametrics/downloader.rb', line 87

def find_board_ids
  ids = @download_config.project_config.board_configs.collect(&:id)
  raise 'Board ids must be specified' if ids.empty?

  ids
end

#identify_other_issues_to_be_downloaded(raw_issue:, board:) ⇒ Object 



94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/jirametrics/downloader.rb', line 94

def identify_other_issues_to_be_downloaded raw_issue:, board:
  issue = Issue.new raw: raw_issue, board: board
  @issue_keys_downloaded_in_current_run << issue.key

  # Parent
  parent_key = issue.parent_key(project_config: @download_config.project_config)
  @issue_keys_pending_download << parent_key if parent_key

  # Sub-tasks
  issue.raw['fields']['subtasks']&.each do |raw_subtask|
    @issue_keys_pending_download << raw_subtask['key']
  end
end

#load_metadata ⇒ Object 



202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/jirametrics/downloader.rb', line 202

def load_metadata
  # If we've never done a download before then this file won't be there. That's ok.
  hash = file_system.load_json(metadata_pathname, fail_on_error: false)
  return if hash.nil?

  # Only use the saved metadata if the version number is the same one that we're currently using.
  # If the cached data is in an older format then we're going to throw most of it away.
  @cached_data_format_is_current = (hash['version'] || 0) == CURRENT_METADATA_VERSION
  if @cached_data_format_is_current
    hash.each do |key, value|
      value = Date.parse(value) if value.is_a?(String) && value =~ /^\d{4}-\d{2}-\d{2}$/
      @metadata[key] = value
    end

    # If rolling_date_count has changed, we may be missing data outside the previous range,
    # so force a full re-download.
    if @metadata['rolling_date_count'] != @download_config.rolling_date_count
      log '  rolling_date_count has changed. Forcing a full download.', both: true
      @cached_data_format_is_current = false
      @metadata = {}
    end
  end

  # Even if this is the old format, we want to obey this one tag
  @metadata['no-download'] = hash['no-download'] if hash['no-download']
end

#log(text, both: false) ⇒ Object 



83
84
85
 
# File 'lib/jirametrics/downloader.rb', line 83

def log text, both: false
  @file_system.log text, also_write_to_stderr: both
end

#make_jql(filter_id:, today: nil) ⇒ Object 



279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
 
# File 'lib/jirametrics/downloader.rb', line 279

def make_jql filter_id:, today: nil
  today ||= today_in_project_timezone
  segments = []
  segments << "filter=#{filter_id}"

  start_date = @download_config.start_date today: today

  if start_date
    @download_date_range = start_date..today.to_date
    @start_date_in_query = @download_date_range.begin

    # Catch-all to pick up anything that's been around since before the range started but hasn't
    # had an update during the range.
    catch_all = '((status changed OR Sprint is not EMPTY) AND statusCategory != Done)'

    # Pick up any issues that had a status change in the range
    start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
    # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
    find_in_range = %(updated >= "#{start_date_text} 00:00")

    segments << "(#{find_in_range} OR #{catch_all})"
  end

  segments.join ' AND '
end

#metadata_pathname ⇒ Object 



198
199
200
 
# File 'lib/jirametrics/downloader.rb', line 198

def metadata_pathname
  File.join(@target_path, "#{file_prefix}_meta.json")
end

#remove_old_files ⇒ Object 



258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
 
# File 'lib/jirametrics/downloader.rb', line 258

def remove_old_files
  Dir.foreach @target_path do |file|
    next unless file.match?(/^#{file_prefix}_\d+\.json$/)
    next if file == "#{file_prefix}_status_history.json"

    File.unlink File.join(@target_path, file)
  end

  return if @cached_data_format_is_current

  # Also throw away all the previously downloaded issues.
  path = File.join(@target_path, "#{file_prefix}_issues")
  return unless File.exist? path

  Dir.foreach path do |file|
    next unless file.match?(/\.json$/)

    File.unlink File.join(path, file)
  end
end

#run ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/jirametrics/downloader.rb', line 57

def run
  log '', both: true
  log @download_config.project_config.name, both: true

  load_metadata

  if @metadata['no-download']
    log '  Skipping download. Found no-download in meta file', both: true
    return
  end

  # board_ids = @download_config.board_ids

  remove_old_files
  update_status_history_file
  download_statuses
  find_board_ids.each do |id|
    board = download_board_configuration board_id: id
    download_issues board: board
  end
  download_users

  save_metadata
  download_github_prs if @download_config.github_repos.any?
end

#save_metadata ⇒ Object 



237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'lib/jirametrics/downloader.rb', line 237

def save_metadata
  @metadata['version'] = CURRENT_METADATA_VERSION
  @metadata['rolling_date_count'] = @download_config.rolling_date_count
  @metadata['date_start_from_last_query'] = @start_date_in_query if @start_date_in_query

  if @download_date_range.nil?
    log "Making up a date range in meta since one wasn't specified. You'll want to change that.", both: true
    today = today_in_project_timezone
    @download_date_range = (today - 7)..today
  end

  @metadata['earliest_date_start'] = @download_date_range.begin if @metadata['earliest_date_start'].nil?

  @metadata['date_start'] = @download_date_range.begin
  @metadata['date_end'] = @download_date_range.end

  @metadata['jira_url'] = @jira_url

  @file_system.save_json json: @metadata, filename: metadata_pathname
end

#timezone_offset ⇒ Object 



229
230
231
 
# File 'lib/jirametrics/downloader.rb', line 229

def timezone_offset
  @download_config.project_config.exporter.timezone_offset
end

#today_in_project_timezone ⇒ Object 



233
234
235
 
# File 'lib/jirametrics/downloader.rb', line 233

def today_in_project_timezone
  Time.now.getlocal(timezone_offset).to_date
end

#update_status_history_file ⇒ Object 



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/jirametrics/downloader.rb', line 130

def update_status_history_file
  status_filename = File.join(@target_path, "#{file_prefix}_statuses.json")
  return unless file_system.file_exist? status_filename

  status_json = file_system.load_json(status_filename)

  history_filename = File.join(@target_path, "#{file_prefix}_status_history.json")
  history_json = file_system.load_json(history_filename) if file_system.file_exist? history_filename

  if history_json
    file_system.log '  Updating status history file', also_write_to_stderr: true
  else
    file_system.log '  Creating status history file', also_write_to_stderr: true
    history_json = []
  end

  status_json.each do |status_item|
    id = status_item['id']
    history_item = history_json.find { |s| s['id'] == id }
    history_json.delete(history_item) if history_item
    history_json << status_item
  end

  file_system.save_json(filename: history_filename, json: history_json)
end