Class: Downloader

Inherits:
Object
  • Object
show all
Defined in:
lib/jirametrics/downloader.rb

Direct Known Subclasses

DownloaderForCloud, DownloaderForDataCenter

Constant Summary collapse

CURRENT_METADATA_VERSION = 
5

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(download_config:, file_system:, jira_gateway:, github_pr_cache: {}) ⇒ Downloader

Returns a new instance of Downloader.



46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/jirametrics/downloader.rb', line 46

def initialize download_config:, file_system:, jira_gateway:, github_pr_cache: {}
  @metadata = {}
  @download_config = download_config
  @target_path = @download_config.project_config.target_path
  @file_system = file_system
  @jira_gateway = jira_gateway
  @github_pr_cache = github_pr_cache
  @board_id_to_filter_id = {}

  @issue_keys_downloaded_in_current_run = []
  @issue_keys_pending_download = []
end

Instance Attribute Details

#board_id_to_filter_id ⇒ Object (readonly)

For testing only



34
35
36
 
# File 'lib/jirametrics/downloader.rb', line 34

def board_id_to_filter_id
  @board_id_to_filter_id
end

#file_system ⇒ Object (readonly)

Returns the value of attribute file_system.



31
32
33
 
# File 'lib/jirametrics/downloader.rb', line 31

def file_system
  @file_system
end

#metadata ⇒ Object

Returns the value of attribute metadata.



30
31
32
 
# File 'lib/jirametrics/downloader.rb', line 30

def metadata
  @metadata
end

#start_date_in_query ⇒ Object (readonly)

For testing only



34
35
36
 
# File 'lib/jirametrics/downloader.rb', line 34

def start_date_in_query
  @start_date_in_query
end

Class Method Details

.create(download_config:, file_system:, jira_gateway:, github_pr_cache: {}) ⇒ Object 



36
37
38
39
40
41
42
43
44
 
# File 'lib/jirametrics/downloader.rb', line 36

def self.create download_config:, file_system:, jira_gateway:, github_pr_cache: {}
  is_cloud = jira_gateway.settings['jira_cloud'] || jira_gateway.cloud?
  (is_cloud ? DownloaderForCloud : DownloaderForDataCenter).new(
    download_config: download_config,
    file_system: file_system,
    jira_gateway: jira_gateway,
    github_pr_cache: github_pr_cache
  )
end

Instance Method Details

#download_board_configuration(board_id:) ⇒ Object 



176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/jirametrics/downloader.rb', line 176

def download_board_configuration board_id:
  log "  Downloading board configuration for board #{board_id}", both: true
  json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/configuration"

  @file_system.save_json(
    json: json,
    filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_configuration.json")
  )

  # We have a reported bug that blew up on this line. Moved it after the save so we can
  # actually look at the returned json.
  @board_id_to_filter_id[board_id] = json['filter']['id'].to_i

  if json['type'] == 'simple'
    features_json = download_features board_id: board_id
    if BoardFeature.from_raw(features_json).any? { |f| f.name == 'jsw.agility.sprints' && f.enabled? }
      download_sprints board_id: board_id
    end
  end
  download_sprints board_id: board_id if json['type'] == 'scrum'
  # TODO: Should be passing actual statuses, not empty list
  Board.new raw: json, possible_statuses: StatusCollection.new
end

#download_features(board_id:) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
 
# File 'lib/jirametrics/downloader.rb', line 200

def download_features board_id:
  log "  Downloading features for board #{board_id}", both: true
  json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/features"

  @file_system.save_json(
    json: json,
    filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_features.json")
  )
  json
end

#download_github_prs ⇒ Object 



335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
 
# File 'lib/jirametrics/downloader.rb', line 335

def download_github_prs
  project_keys = extract_project_keys_from_downloaded_issues
  if project_keys.empty?
    log '  No project keys found in downloaded issues, skipping GitHub PR download', both: true
    return
  end

  prs = @download_config.github_repos.flat_map do |repo|
    GithubGateway.new(
      repo: repo,
      project_keys: project_keys,
      file_system: @file_system,
      raw_pr_cache: @github_pr_cache
    ).fetch_pull_requests(since: @download_date_range&.begin)
  end

  @file_system.save_json(
    json: prs.map(&:raw),
    filename: File.join(@target_path, "#{file_prefix}_github_prs.json")
  )
end

#download_sprints(board_id:) ⇒ Object 



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/jirametrics/downloader.rb', line 211

def download_sprints board_id:
  log "  Downloading sprints for board #{board_id}", both: true
  max_results = 100
  start_at = 0
  is_last = false

  while is_last == false
    json = @jira_gateway.call_url relative_url: "/rest/agile/1.0/board/#{board_id}/sprint?" \
      "maxResults=#{max_results}&startAt=#{start_at}"

    @file_system.save_json(
      json: json,
      filename: File.join(@target_path, "#{file_prefix}_board_#{board_id}_sprints_#{start_at}.json")
    )
    is_last = json['isLast']
    max_results = json['maxResults']
    if json['values']
      start_at += json['values'].size
    else
      log "  No sprints found for board #{board_id}"
    end
  end
end

#download_statuses ⇒ Object 



128
129
130
131
132
133
134
135
136
 
# File 'lib/jirametrics/downloader.rb', line 128

def download_statuses
  log '  Downloading all statuses', both: true
  json = @jira_gateway.call_url relative_url: '/rest/api/2/status'

  @file_system.save_json(
    json: json,
    filename: File.join(@target_path, "#{file_prefix}_statuses.json")
  )
end

#download_users ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/jirametrics/downloader.rb', line 138

def download_users
  return unless @jira_gateway.cloud?

  log '  Downloading all users', both: true
  json = @jira_gateway.call_url relative_url: '/rest/api/2/users'

  @file_system.save_json(
    json: json,
    filename: File.join(@target_path, "#{file_prefix}_users.json")
  )
end

#end_progress ⇒ Object 



103
104
105
 
# File 'lib/jirametrics/downloader.rb', line 103

def end_progress
  @file_system.end_progress
end

#extract_project_keys_from_downloaded_issues ⇒ Object 



357
358
359
360
361
362
363
364
365
366
 
# File 'lib/jirametrics/downloader.rb', line 357

def extract_project_keys_from_downloaded_issues
  path = File.join(@target_path, "#{file_prefix}_issues")
  return [] unless @file_system.dir_exist?(path)

  keys = []
  @file_system.foreach(path) do |filename|
    keys << filename.split('-').first if filename.match?(/^[A-Z][A-Z_0-9]+-\d+-\d+\.json$/)
  end
  keys.uniq
end

#file_prefix ⇒ Object 



368
369
370
 
# File 'lib/jirametrics/downloader.rb', line 368

def file_prefix
  @download_config.project_config.get_file_prefix
end

#find_board_ids ⇒ Object 



107
108
109
110
111
112
 
# File 'lib/jirametrics/downloader.rb', line 107

def find_board_ids
  ids = @download_config.project_config.board_configs.collect(&:id)
  raise 'Board ids must be specified' if ids.empty?

  ids
end

#identify_other_issues_to_be_downloaded(raw_issue:, board:) ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/jirametrics/downloader.rb', line 114

def identify_other_issues_to_be_downloaded raw_issue:, board:
  issue = Issue.new raw: raw_issue, board: board
  @issue_keys_downloaded_in_current_run << issue.key

  # Parent
  parent_key = issue.parent_key(project_config: @download_config.project_config)
  @issue_keys_pending_download << parent_key if parent_key

  # Sub-tasks
  issue.raw['fields']['subtasks']&.each do |raw_subtask|
    @issue_keys_pending_download << raw_subtask['key']
  end
end

#load_metadata ⇒ Object 



239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
 
# File 'lib/jirametrics/downloader.rb', line 239

def load_metadata
  # If we've never done a download before then this file won't be there. That's ok.
  hash = file_system.load_json(metadata_pathname, fail_on_error: false)
  return if hash.nil?

  # Only use the saved metadata if the version number is the same one that we're currently using.
  # If the cached data is in an older format then we're going to throw most of it away.
  @cached_data_format_is_current = (hash['version'] || 0) == CURRENT_METADATA_VERSION
  if @cached_data_format_is_current
    hash.each do |key, value|
      value = Date.parse(value) if value.is_a?(String) && value =~ /^\d{4}-\d{2}-\d{2}$/
      @metadata[key] = value
    end

  end

  # Even if this is the old format, we want to obey this one tag
  @metadata['no-download'] = hash['no-download'] if hash['no-download']
end

#log(text, both: false) ⇒ Object 



86
87
88
 
# File 'lib/jirametrics/downloader.rb', line 86

def log text, both: false
  @file_system.log text, also_write_to_stderr: both
end

#log_start(text) ⇒ Object 



90
91
92
 
# File 'lib/jirametrics/downloader.rb', line 90

def log_start text
  @file_system.log_start text
end

#make_jql(filter_id:, today: nil) ⇒ Object 



309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
 
# File 'lib/jirametrics/downloader.rb', line 309

def make_jql filter_id:, today: nil
  today ||= today_in_project_timezone
  segments = []
  segments << "filter=#{filter_id}"

  start_date = @download_config.start_date today: today

  if start_date
    @download_date_range = start_date..today.to_date
    @start_date_in_query = @download_date_range.begin

    # Catch-all to pick up anything that's been around since before the range started but hasn't
    # had an update during the range.
    catch_all = '((status changed OR Sprint is not EMPTY) AND statusCategory != Done)'

    # Pick up any issues that had a status change in the range
    start_date_text = @start_date_in_query.strftime '%Y-%m-%d'
    # find_in_range = %((status changed DURING ("#{start_date_text} 00:00","#{end_date_text} 23:59")))
    find_in_range = %(updated >= "#{start_date_text} 00:00")

    segments << "(#{find_in_range} OR #{catch_all})"
  end

  segments.join ' AND '
end

#metadata_pathname ⇒ Object 



235
236
237
 
# File 'lib/jirametrics/downloader.rb', line 235

def metadata_pathname
  File.join(@target_path, "#{file_prefix}_meta.json")
end

#progress_dot(message = nil) ⇒ Object 



98
99
100
101
 
# File 'lib/jirametrics/downloader.rb', line 98

def progress_dot message = nil
  @file_system.log message if message
  @file_system.progress_dot
end

#remove_old_files ⇒ Object 



288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
 
# File 'lib/jirametrics/downloader.rb', line 288

def remove_old_files
  Dir.foreach @target_path do |file|
    next unless file.match?(/^#{file_prefix}_\d+\.json$/)
    next if file == "#{file_prefix}_status_history.json"

    File.unlink File.join(@target_path, file)
  end

  return if @cached_data_format_is_current

  # Also throw away all the previously downloaded issues.
  path = File.join(@target_path, "#{file_prefix}_issues")
  return unless File.exist? path

  Dir.foreach path do |file|
    next unless file.match?(/\.json$/)

    File.unlink File.join(path, file)
  end
end

#run ⇒ Object 



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/jirametrics/downloader.rb', line 59

def run
  log '', both: true
  log @download_config.project_config.name, both: true

  load_metadata

  if @metadata['no-download']
    log '  Skipping download. Found no-download in meta file', both: true
    return
  end

  # board_ids = @download_config.board_ids

  remove_old_files
  update_status_history_file
  download_statuses
  find_board_ids.each do |id|
    board = download_board_configuration board_id: id
    board.project_config = @download_config.project_config
    download_issues board: board
  end
  download_users

  save_metadata
  download_github_prs if @download_config.github_repos.any?
end

#save_metadata ⇒ Object 



267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
 
# File 'lib/jirametrics/downloader.rb', line 267

def save_metadata
  @metadata['version'] = CURRENT_METADATA_VERSION
  @metadata['rolling_date_count'] = @download_config.rolling_date_count
  @metadata['date_start_from_last_query'] = @start_date_in_query if @start_date_in_query

  if @download_date_range.nil?
    log "Making up a date range in meta since one wasn't specified. You'll want to change that.", both: true
    today = today_in_project_timezone
    @download_date_range = (today - 7)..today
  end

  @metadata['earliest_date_start'] = @download_date_range.begin if @metadata['earliest_date_start'].nil?

  @metadata['date_start'] = @download_date_range.begin
  @metadata['date_end'] = @download_date_range.end

  @metadata['jira_url'] = @jira_url

  @file_system.save_json json: @metadata, filename: metadata_pathname
end

#start_progress ⇒ Object 



94
95
96
 
# File 'lib/jirametrics/downloader.rb', line 94

def start_progress
  @file_system.start_progress
end

#timezone_offset ⇒ Object 



259
260
261
 
# File 'lib/jirametrics/downloader.rb', line 259

def timezone_offset
  @download_config.project_config.exporter.timezone_offset
end

#today_in_project_timezone ⇒ Object 



263
264
265
 
# File 'lib/jirametrics/downloader.rb', line 263

def today_in_project_timezone
  Time.now.getlocal(timezone_offset).to_date
end

#update_status_history_file ⇒ Object 



150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/jirametrics/downloader.rb', line 150

def update_status_history_file
  status_filename = File.join(@target_path, "#{file_prefix}_statuses.json")
  return unless file_system.file_exist? status_filename

  status_json = file_system.load_json(status_filename)

  history_filename = File.join(@target_path, "#{file_prefix}_status_history.json")
  history_json = file_system.load_json(history_filename) if file_system.file_exist? history_filename

  if history_json
    file_system.log '  Updating status history file', also_write_to_stderr: true
  else
    file_system.log '  Creating status history file', also_write_to_stderr: true
    history_json = []
  end

  status_json.each do |status_item|
    id = status_item['id']
    history_item = history_json.find { |s| s['id'] == id }
    history_json.delete(history_item) if history_item
    history_json << status_item
  end

  file_system.save_json(filename: history_filename, json: history_json)
end