Module: SchwabRb::PriceHistory::Downloader

Defined in:
lib/schwab_rb/price_history/downloader.rb

Constant Summary collapse

INDEX_API_SYMBOLS = 
%w[
  COMPX DJX MID NDX OEX RUT SPX VIX VIX9D VIX1D XSP
].freeze
SUPPORTED_FORMATS = 
%w[csv json].freeze
FREQUENCY_ALIASES = 
{
  "1min" => {
    label: "1min",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::MINUTE,
    frequency: SchwabRb::PriceHistory::Frequencies::EVERY_MINUTE,
    period_type: SchwabRb::PriceHistory::PeriodTypes::DAY,
    period: SchwabRb::PriceHistory::Periods::ONE_DAY
  },
  "5min" => {
    label: "5min",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::MINUTE,
    frequency: SchwabRb::PriceHistory::Frequencies::EVERY_FIVE_MINUTES,
    period_type: SchwabRb::PriceHistory::PeriodTypes::DAY,
    period: SchwabRb::PriceHistory::Periods::ONE_DAY
  },
  "10min" => {
    label: "10min",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::MINUTE,
    frequency: SchwabRb::PriceHistory::Frequencies::EVERY_TEN_MINUTES,
    period_type: SchwabRb::PriceHistory::PeriodTypes::DAY,
    period: SchwabRb::PriceHistory::Periods::ONE_DAY
  },
  "15min" => {
    label: "15min",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::MINUTE,
    frequency: SchwabRb::PriceHistory::Frequencies::EVERY_FIFTEEN_MINUTES,
    period_type: SchwabRb::PriceHistory::PeriodTypes::DAY,
    period: SchwabRb::PriceHistory::Periods::ONE_DAY
  },
  "30min" => {
    label: "30min",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::MINUTE,
    frequency: SchwabRb::PriceHistory::Frequencies::EVERY_THIRTY_MINUTES,
    period_type: SchwabRb::PriceHistory::PeriodTypes::DAY,
    period: SchwabRb::PriceHistory::Periods::ONE_DAY
  },
  "day" => {
    label: "day",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::DAILY,
    frequency: SchwabRb::PriceHistory::Frequencies::DAILY,
    period_type: SchwabRb::PriceHistory::PeriodTypes::YEAR,
    period: SchwabRb::PriceHistory::Periods::TWENTY_YEARS
  },
  "week" => {
    label: "week",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::WEEKLY,
    frequency: SchwabRb::PriceHistory::Frequencies::WEEKLY,
    period_type: SchwabRb::PriceHistory::PeriodTypes::YEAR,
    period: SchwabRb::PriceHistory::Periods::TWENTY_YEARS
  },
  "month" => {
    label: "month",
    frequency_type: SchwabRb::PriceHistory::FrequencyTypes::MONTHLY,
    frequency: SchwabRb::PriceHistory::Frequencies::MONTHLY,
    period_type: SchwabRb::PriceHistory::PeriodTypes::YEAR,
    period: SchwabRb::PriceHistory::Periods::TWENTY_YEARS
  }
}.freeze

Class Method Summary collapse

Class Method Details

.api_symbol(symbol) ⇒ Object 



112
113
114
115
116
117
118
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 112

def api_symbol(symbol)
  raw_symbol = symbol.to_s.strip
  return raw_symbol if raw_symbol.start_with?("$", "/")
  return "$#{raw_symbol}" if INDEX_API_SYMBOLS.include?(raw_symbol.upcase)

  raw_symbol
end

.business_dates_in_range(start_date, end_date) ⇒ Object 



215
216
217
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 215

def business_dates_in_range(start_date, end_date)
  (start_date..end_date).select { |date| (1..5).cover?(date.wday) }
end

.cached_range_covers?(response, start_date, end_date, frequency) ⇒ Boolean

Returns:

  • (Boolean) 


191
192
193
194
195
196
197
198
199
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 191

def cached_range_covers?(response, start_date, end_date, frequency)
  return false unless response

  dates = requested_candle_dates(response, start_date, end_date)
  return false if dates.empty?
  return daily_range_covered?(dates, start_date, end_date) if frequency == "day"

  start_date >= dates.first && end_date <= dates.last
end

.candle_dates(response) ⇒ Object 



201
202
203
204
205
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 201

def candle_dates(response)
  Array(response[:candles]).map do |candle|
    Time.at(candle.fetch(:datetime) / 1000.0).utc.to_date
  end.sort
end

.canonical_output_path(directory:, symbol:, frequency:, format:) ⇒ Object 



103
104
105
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 103

def canonical_output_path(directory:, symbol:, frequency:, format:)
  File.join(directory, "#{sanitize_symbol(symbol)}_#{frequency}.#{format}")
end

.daily_range_covered?(cached_dates, start_date, end_date) ⇒ Boolean

Returns:

  • (Boolean) 


211
212
213
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 211

def daily_range_covered?(cached_dates, start_date, end_date)
  business_dates_in_range(start_date, end_date).all? { |date| cached_dates.include?(date) }
end

.load_cached_price_history(path, format) ⇒ Object 



143
144
145
146
147
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 143

def load_cached_price_history(path, format)
  return unless File.exist?(path)

  parse_price_history_file(path, format)
end

.merge_price_history_responses(left, right, fallback_symbol) ⇒ Object 



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 219

def merge_price_history_responses(left, right, fallback_symbol)
  return normalize_price_history_response(right, fallback_symbol) unless left
  return normalize_price_history_response(left, fallback_symbol) unless right

  merged_candles = Array(left[:candles]) + Array(right[:candles])
  deduped_candles = merged_candles.each_with_object({}) do |candle, by_datetime|
    by_datetime[candle.fetch(:datetime)] = candle.transform_keys(&:to_sym)
  end

  {
    symbol: left[:symbol] || right[:symbol] || fallback_symbol,
    empty: deduped_candles.empty?,
    candles: deduped_candles.values.sort_by { |candle| candle.fetch(:datetime) }
  }
end

.normalize_price_history_response(response, fallback_symbol) ⇒ Object 



235
236
237
238
239
240
241
242
243
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 235

def normalize_price_history_response(response, fallback_symbol)
  {
    symbol: response[:symbol] || fallback_symbol,
    empty: Array(response[:candles]).empty?,
    candles: Array(response[:candles]).map { |candle| candle.transform_keys(&:to_sym) }.sort_by do |candle|
      candle.fetch(:datetime)
    end
  }
end

.parse_price_history_csv(path) ⇒ Object 



162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 162

def parse_price_history_csv(path)
  candles = CSV.read(path, headers: true).map do |row|
    {
      datetime: Time.iso8601(row.fetch("datetime")).to_i * 1000,
      open: row.fetch("open").to_f,
      high: row.fetch("high").to_f,
      low: row.fetch("low").to_f,
      close: row.fetch("close").to_f,
      volume: row.fetch("volume").to_i
    }
  end

  {
    symbol: File.basename(path).split("_").first,
    empty: candles.empty?,
    candles: candles
  }
end

.parse_price_history_file(path, format) ⇒ Object 



149
150
151
152
153
154
155
156
157
158
159
160
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 149

def parse_price_history_file(path, format)
  case format
  when "json"
    symbolize_price_history_payload(JSON.parse(File.read(path)))
  when "csv"
    parse_price_history_csv(path)
  else
    raise ArgumentError, "Unsupported format `#{format}`."
  end
rescue JSON::ParserError => e
  raise ArgumentError, "Unable to parse cached price history at #{path}: #{e.message}"
end

.requested_candle_dates(response, start_date, end_date) ⇒ Object 



207
208
209
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 207

def requested_candle_dates(response, start_date, end_date)
  candle_dates(response).select { |date| date >= start_date && date <= end_date }
end

.resolve(client:, symbol:, start_date:, end_date:, directory:, frequency:, format:, need_extended_hours_data:, need_previous_close:, period_type: nil, period: nil) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 76

def resolve(client:, symbol:, start_date:, end_date:, directory:, frequency:, format:, need_extended_hours_data:, need_previous_close:, period_type: nil, period: nil)
  config = FREQUENCY_ALIASES.fetch(frequency)
  path = canonical_output_path(directory: directory, symbol: symbol, frequency: frequency, format: format)
  existing_response = load_cached_price_history(path, format)
  response = existing_response

  unless cached_range_covers?(existing_response, start_date, end_date, frequency)
    FileUtils.mkdir_p(directory)
    downloaded = client.get_price_history(
      api_symbol(symbol),
      period_type: period_type || config.fetch(:period_type),
      period: period || config.fetch(:period),
      frequency_type: config.fetch(:frequency_type),
      frequency: config.fetch(:frequency),
      start_datetime: start_date,
      end_datetime: end_date,
      need_extended_hours_data: need_extended_hours_data,
      need_previous_close: need_previous_close,
      return_data_objects: false
    )
    response = merge_price_history_responses(response, downloaded, symbol)
  end

  File.write(path, serialized_payload(response, format))
  [response, path]
end

.sanitize_symbol(symbol) ⇒ Object 



107
108
109
110
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 107

def sanitize_symbol(symbol)
  sanitized_symbol = symbol.to_s.gsub(/[^a-zA-Z0-9]+/, "_").gsub(/\A_+|_+\z/, "")
  sanitized_symbol.empty? ? "symbol" : sanitized_symbol
end

.serialized_payload(response, format) ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 120

def serialized_payload(response, format)
  case format
  when "json"
    JSON.pretty_generate(response)
  when "csv"
    CSV.generate do |csv|
      csv << %w[datetime open high low close volume]
      Array(response[:candles]).each do |candle|
        csv << [
          Time.at(candle.fetch(:datetime) / 1000.0).utc.iso8601,
          candle[:open],
          candle[:high],
          candle[:low],
          candle[:close],
          candle[:volume]
        ]
      end
    end
  else
    raise ArgumentError, "Unsupported format `#{format}`."
  end
end

.symbolize_price_history_payload(payload) ⇒ Object 



181
182
183
184
185
186
187
188
189
 
# File 'lib/schwab_rb/price_history/downloader.rb', line 181

def symbolize_price_history_payload(payload)
  {
    symbol: payload["symbol"] || payload[:symbol],
    empty: payload["empty"].nil? ? payload[:empty] : payload["empty"],
    candles: Array(payload["candles"] || payload[:candles]).map do |candle|
      candle.transform_keys(&:to_sym)
    end
  }
end