Class: GitLab::Exporter::Database::ZoektCollector

Inherits:

Base

• Object
• Base
• GitLab::Exporter::Database::ZoektCollector

Defined in:

lib/gitlab_exporter/database/zoekt.rb

Overview

A helper class to collect zoekt metrics. rubocop:disable Metrics/ClassLength

Constant Summary

ZOEKT_TASKS_PROCESSING_QUERY =

Query to get processing zoekt_tasks distribution by zoekt_node_id

<<~SQL.freeze
  WITH task_counts AS (
    SELECT
      zoekt_node_id,
      COUNT(*) AS count
    FROM
      zoekt_tasks
    WHERE
      perform_at <= $1
      AND state IN (0, 1)
    GROUP BY
      zoekt_node_id
  )
  SELECT
    n.id AS node_id,
    n.metadata ->> 'name' AS node_name,
    COALESCE(tc.count, 0) AS task_count
  FROM
    zoekt_nodes n
  LEFT JOIN
    task_counts tc ON n.id = tc.zoekt_node_id
SQL

ZOEKT_NODES_QUERY =

<<~SQL.freeze
  SELECT schema_version, id, metadata ->> 'name' AS node_name FROM zoekt_nodes
SQL

ZOEKT_NODES_STATUS_QUERY =

<<~SQL.freeze
  SELECT id, metadata ->> 'name' AS node_name,
    CASE
      WHEN last_seen_at < NOW() - INTERVAL '2 minute' THEN 0
      ELSE 1
    END AS status
  FROM zoekt_nodes;
SQL

ZOEKT_REPOSITORY_SCHEMA_VERSION_QUERY =

<<~SQL.freeze
  SELECT
    COUNT(*)
  FROM
    zoekt_repositories
  INNER JOIN
    zoekt_indices
    ON zoekt_repositories.zoekt_index_id = zoekt_indices.id
  WHERE
    zoekt_indices.zoekt_node_id = $1
    AND zoekt_repositories.schema_version < $2
    AND zoekt_repositories.state = 10;
SQL

ZOEKT_ENABLED_QUERY =

<<~SQL.freeze
  SELECT
    zoekt_settings ->> 'zoekt_indexing_enabled' AS zoekt_indexing_enabled
  FROM application_settings
  ORDER BY ID DESC
  LIMIT 1
SQL

ZOEKT_NODE_STORAGE_QUERY =

unclaimed_storage_bytes and storage_percent_used are not real columns on zoekt_nodes; they are derived in Search::Zoekt::Node. We compute them in SQL here to avoid loading every row through ActiveRecord.

<<~SQL.freeze
  SELECT
    zn.id,
    zn.metadata ->> 'name' AS node_name,
    (zn.usable_storage_bytes - COALESCE(zi.reserved_total, 0))::bigint
      AS unclaimed_storage_bytes,
    CASE
      WHEN zn.total_bytes > 0
        THEN zn.used_bytes::float8 / zn.total_bytes::float8
      ELSE 0
    END AS storage_percent_used
  FROM zoekt_nodes zn
  LEFT JOIN (
    SELECT zoekt_node_id, SUM(reserved_storage_bytes) AS reserved_total
    FROM zoekt_indices
    GROUP BY zoekt_node_id
  ) zi ON zi.zoekt_node_id = zn.id
SQL

ZOEKT_REPOSITORIES_STATE_QUERY =

<<~SQL.freeze
  SELECT state, COUNT(*) FROM zoekt_repositories GROUP BY state
SQL

ZOEKT_INDICES_STATE_QUERY =

<<~SQL.freeze
  SELECT state, COUNT(*) FROM zoekt_indices GROUP BY state
SQL

ZOEKT_INDICES_WATERMARK_QUERY =

<<~SQL.freeze
  SELECT watermark_level, COUNT(*) FROM zoekt_indices GROUP BY watermark_level
SQL

ZOEKT_INDICES_STORAGE_QUERY =

<<~SQL.freeze
  SELECT
    zoekt_indices.id,
    zoekt_indices.reserved_storage_bytes,
    zoekt_indices.used_storage_bytes,
    zoekt_nodes.metadata ->> 'name' AS node_name
  FROM
    zoekt_indices
  INNER JOIN
    zoekt_nodes ON zoekt_indices.zoekt_node_id = zoekt_nodes.id
SQL

ZOEKT_NODE_ENABLED_NAMESPACES_QUERY =

<<~SQL.freeze
  SELECT
    zn.id AS node_id,
    zn.metadata ->> 'name' AS node_name,
    COUNT(DISTINCT zi.zoekt_enabled_namespace_id) AS count
  FROM zoekt_nodes zn
  LEFT JOIN zoekt_indices zi ON zi.zoekt_node_id = zn.id
  GROUP BY zn.id, zn.metadata ->> 'name'
SQL

ZOEKT_NODE_TASKS_QUERY =

<<~SQL.freeze
  SELECT
    zn.id AS node_id,
    zn.metadata ->> 'name' AS node_name,
    zt.state,
    COUNT(*) AS count
  FROM zoekt_nodes zn
  LEFT JOIN zoekt_tasks zt ON zt.zoekt_node_id = zn.id
  GROUP BY zn.id, zn.metadata ->> 'name', zt.state
SQL

ZOEKT_INDICES_STALE_USED_STORAGE_QUERY =

<<~SQL.freeze
  SELECT COUNT(*) AS count FROM zoekt_indices WHERE last_indexed_at >= used_storage_bytes_updated_at
SQL

ZOEKT_TASK_STATES =

{
  0 => "pending",
  1 => "processing",
  10 => "done",
  250 => "skipped",
  255 => "failed",
  256 => "orphaned"
}.freeze

Constants inherited from Base

Base::POOL_SIZE, Base::POOL_TIMEOUT

Instance Method Summary

• #run ⇒ Object

Methods inherited from Base

configure_type_map_for_results, connection_pool, #connection_pool, #initialize, #with_connection_pool

Constructor Details

This class inherits a constructor from GitLab::Exporter::Database::Base

Instance Method Details

#run ⇒ Object

# File 'lib/gitlab_exporter/database/zoekt.rb', line 146

def run
  return {} unless zoekt_indexing_enabled?

  {
    task_processing_query_result: execute(ZOEKT_TASKS_PROCESSING_QUERY, [Time.now.utc]),
    repositories_schema_version_query_result: repositories_schema_version_query_result,
    zoekt_nodes_status_query_result: execute(ZOEKT_NODES_STATUS_QUERY),
    node_storage_query_result: execute(ZOEKT_NODE_STORAGE_QUERY),
    repositories_state_query_result: execute(ZOEKT_REPOSITORIES_STATE_QUERY),
    indices_state_query_result: execute(ZOEKT_INDICES_STATE_QUERY),
    indices_watermark_query_result: execute(ZOEKT_INDICES_WATERMARK_QUERY),
    indices_storage_query_result: execute(ZOEKT_INDICES_STORAGE_QUERY),
    node_enabled_namespaces_query_result: execute(ZOEKT_NODE_ENABLED_NAMESPACES_QUERY),
    node_tasks_query_result: execute(ZOEKT_NODE_TASKS_QUERY),
    indices_stale_used_storage_query_result: execute(ZOEKT_INDICES_STALE_USED_STORAGE_QUERY)
  }.compact
end