Class: LlmCostTracker::Dashboard::DataQuality

Inherits:
Object
  • Object
show all
Defined in:
app/services/llm_cost_tracker/dashboard/data_quality.rb

Defined Under Namespace

Classes: StreamingHealthRow, Summary, UnknownPricingRow

Class Method Summary collapse

Class Method Details

.call(scope: LlmCostTracker::Call.all) ⇒ Object 



20
21
22
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 20

def call(scope: LlmCostTracker::Call.all)
  scope.unscope(:order).select(aggregate_selects(scope)).take
end

.component_costs(scope) ⇒ Object 



115
116
117
118
119
120
121
122
123
124
125
126
127
128
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 115

def component_costs(scope)
  line_item_table = LlmCostTracker::CallLineItem.quoted_table_name
  rows = LlmCostTracker::CallLineItem
         .where(unit: "token")
         .joins(:call)
         .merge(scope.unscope(:select, :order, :group))
         .group("#{line_item_table}.kind", "#{line_item_table}.direction",
                "#{line_item_table}.cache_state")
         .pluck(Arel.sql("#{line_item_table}.kind"),
                Arel.sql("#{line_item_table}.direction"),
                Arel.sql("#{line_item_table}.cache_state"),
                Arel.sql("COALESCE(SUM(#{line_item_table}.cost), 0)"))
  index_costs_by_component(rows)
end

.hidden_output_summary(stats) ⇒ Object 



157
158
159
160
161
162
163
164
165
166
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 157

def hidden_output_summary(stats)
  output_tokens = stats.output_tokens.to_i
  return unless output_tokens.positive?

  {
    hidden_output_tokens: stats.hidden_output_tokens.to_i,
    output_tokens: output_tokens,
    share_percent: stats.hidden_output_share.to_f
  }
end

.service_charge_rows(scope) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 63

def service_charge_rows(scope)
  call_table = LlmCostTracker::Call.quoted_table_name
  line_item_table = LlmCostTracker::CallLineItem.quoted_table_name
  relation = LlmCostTracker::CallLineItem
             .where.not(unit: "token")
             .joins(:call)
             .merge(scope.unscope(:select, :order))

  relation
    .group("#{call_table}.provider", "#{line_item_table}.kind", "#{line_item_table}.cost_status")
    .order(Arel.sql("COALESCE(SUM(#{line_item_table}.cost), 0) DESC"), Arel.sql("COUNT(*) DESC"))
    .select(
      "#{call_table}.provider AS provider",
      "#{line_item_table}.kind AS component",
      "#{line_item_table}.cost_status AS cost_status",
      "COUNT(*) AS charges_count",
      "COALESCE(SUM(#{line_item_table}.quantity), 0) AS quantity",
      "COALESCE(SUM(#{line_item_table}.cost), 0) AS total_cost"
    )
    .limit(10)
end

.streaming_health_rows(scope, total_streaming:) ⇒ Object 



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 130

def streaming_health_rows(scope, total_streaming:)
  return [] unless total_streaming.positive?

  unknown_predicate = "usage_source = 'unknown' OR usage_source IS NULL"
  rows = scope.unscope(:select, :order, :group)
              .where(stream: true)
              .group(:provider)
              .order(Arel.sql("COUNT(*) DESC"), :provider)
              .pluck(
                :provider,
                Arel.sql("COUNT(*)"),
                Arel.sql("SUM(CASE WHEN #{unknown_predicate} THEN 1 ELSE 0 END)")
              )

  rows.map do |provider, streams, unknown|
    streams_count = streams.to_i
    unknown_count = unknown.to_i
    StreamingHealthRow.new(
      provider: provider,
      streams: streams_count,
      with_usage: streams_count - unknown_count,
      unknown: unknown_count,
      unknown_share: percentage(unknown_count, streams_count)
    )
  end
end

.summary(stats) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 24

def summary(stats)
  total = stats.total_calls.to_i
  unknown_pricing_count = stats.unknown_pricing_count.to_i
  untagged_calls_count = stats.untagged_calls_count.to_i
  missing_latency_count = stats.missing_latency_count.to_i
  streaming_count = stats.streaming_count.to_i
  streaming_missing_usage = stats.streaming_missing_usage_count.to_i
  missing_provider_response_id_count = stats.missing_provider_response_id_count.to_i
  calls_with_pricing = total - unknown_pricing_count
  tagged_calls = total - untagged_calls_count
  calls_with_latency = total - missing_latency_count
  streams_with_usage = streaming_count - streaming_missing_usage
  calls_with_provider_response_id = total - missing_provider_response_id_count

  Summary.new(
    total, unknown_pricing_count, untagged_calls_count, missing_latency_count, streaming_count,
    streaming_missing_usage, missing_provider_response_id_count, calls_with_pricing, tagged_calls,
    calls_with_latency, streams_with_usage, calls_with_provider_response_id,
    percentage(unknown_pricing_count, total), percentage(untagged_calls_count, total),
    percentage(missing_latency_count, total), percentage(streaming_count, total),
    percentage(streaming_missing_usage, streaming_count), percentage(calls_with_pricing, total),
    percentage(tagged_calls, total), percentage(calls_with_latency, total),
    percentage(streams_with_usage, streaming_count), percentage(calls_with_provider_response_id, total)
  )
end

.unknown_pricing_by_model(scope, total_calls:) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 50

def unknown_pricing_by_model(scope, total_calls:)
  scope.unknown_pricing
       .group(:provider, :model)
       .order(Arel.sql("COUNT(*) DESC"))
       .select("provider, model, COUNT(*) AS calls")
       .limit(10)
       .map do |row|
         calls = row.calls.to_i
         UnknownPricingRow.new(provider: row.provider, model: row.model, calls: calls,
                               share_percent: percentage(calls, total_calls))
       end
end

.usage_rows(stats, component_costs: {}) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'app/services/llm_cost_tracker/dashboard/data_quality.rb', line 85

def usage_rows(stats, component_costs: {})
  billable_tokens = stats.billable_tokens.to_f

  rows = Billing::Components::TOKEN_PRICED.map do |component|
    token_value = stats[component.token_key].to_i

    {
      price_key: component.key,
      token_key: component.token_key,
      cost_key: component.cost_key,
      token_value: token_value,
      cost_value: component_costs[component.key],
      share_percent: percentage(token_value, billable_tokens),
      share_basis: nil
    }
  end

  rows + [
    {
      price_key: nil,
      token_key: :hidden_output_tokens,
      cost_key: nil,
      token_value: stats.hidden_output_tokens.to_i,
      cost_value: nil,
      share_percent: stats.hidden_output_share.to_f,
      share_basis: :output
    }
  ]
end