Class: StructuredDataToSql::Json::Profiles::KhorosApiExport::InputValidator

Inherits:
Object
  • Object
show all
Defined in:
lib/structured_data_to_sql/json/profiles/khoros_api_export.rb

Overview

Validates the converter-facing key contract while records are already streaming through the JSON emitter, so an atomic conversion fails before publishing an unusable dump. Each raw identity (envelope id plus every flattened context field) keeps only its greatest-_sid outcome: an identity digest, the winning canonical id, and any violation message (users add five richness flags). Memory therefore scales with distinct identities at a few dozen bytes each — for identity-dense collections such as messages that is per raw row, so a multi-million-row export costs hundreds of megabytes, not gigabytes.

Constant Summary collapse

MAX_KEY_BYTES = 
255
USER_EMAIL_FIELDS = 
USER_STRING_FIELDS.fetch(:email)
USER_BANNED_FIELD = 
USER_SCALAR_FIELDS.fetch(:banned).first
QUALITY_SIGNAL_TABLES =

Exporter quality signals (contract §9.7): rows that explain what is knowingly missing. Tallied by the first present label field and surfaced as warnings so an empty helper reads as "withheld", not "none".

{
  "errors" => %w[collection phase],
  "gaps" => %w[collection phase]
}.freeze
DIRECT = 
{
  "avatars" => {
    key: %w[id],
    indexed: {
      user_id: %w[id]
    }
  },
  "users" => {
    key: %w[data_id],
    indexed: {
      id: %w[data_id],
      email: %w[data_email data_email_address],
      rank_id: %w[data_rank_id]
    }
  },
  "nodes" => {
    key: %w[data_id],
    indexed: {
      id: %w[data_id],
      parent_id: %w[data_parent_id]
    }
  },
  "ranks" => {
    key: %w[data_id],
    indexed: {
      id: %w[data_id]
    }
  },
  "roles" => {
    key: %w[data_id],
    indexed: {
      id: %w[data_id],
      node_id: %w[data_node_id]
    }
  },
  "messages" => {
    key: %w[data_id],
    indexed: {
      id: %w[data_id],
      conversation_id: %w[data_conversation_id],
      board_id: %w[data_board_id]
    }
  }
}.freeze
OPTIONAL = 
{
  "user_profiles" => {
    required: {
      user_id: %w[id]
    },
    indexed: {
      user_id: %w[id]
    }
  },
  "user_profiles_data_profile" => {
    required: {
      profile_name: %w[name],
      profile_value: %w[unnamed]
    },
    indexed: {
      profile_name: %w[name]
    },
    child_of: "user_profiles",
    normalized_profile_key: true
  },
  "role_users" => {
    required: {
      container_id: %w[context_role_id],
      user_id: %w[id]
    },
    indexed: {
      container_id: %w[context_role_id],
      user_id: %w[id]
    }
  },
  "grouphub_members" => {
    required: {
      container_id: %w[context_grouphub_id],
      user_id: %w[id]
    },
    indexed: {
      container_id: %w[context_grouphub_id],
      user_id: %w[id]
    }
  },
  "badges" => {
    required: {
      badge_id: %w[data_id_unnamed data_id]
    },
    indexed: {
      badge_id: %w[data_id_unnamed data_id]
    }
  },
  "user_badges" => {
    required: {
      user_id: %w[context_user_id],
      badge_id: %w[
        data_badge_id_unnamed
        data_badge_id
        data_id_unnamed
        data_id
      ]
    },
    indexed: {
      user_id: %w[context_user_id],
      badge_id: %w[
        data_badge_id_unnamed
        data_badge_id
        data_id_unnamed
        data_id
      ]
    }
  },
  "kudos" => {
    required: {
      message_id: %w[context_message_id],
      engagement_id: %w[id]
    },
    indexed: {
      message_id: %w[context_message_id],
      engagement_id: %w[id],
      user_id: %w[data_user_id]
    }
  },
  "floated_messages" => {
    required: {
      board_id: %w[context_board_id],
      message_id: %w[data_message_id id]
    },
    indexed: {
      board_id: %w[context_board_id],
      message_id: %w[data_message_id id]
    }
  },
  "labels" => {
    required: {
      source_id: %w[id],
      message_id: %w[context_message_id]
    },
    indexed: {
      source_id: %w[id],
      message_id: %w[context_message_id]
    }
  },
  "tags" => {
    required: {
      source_id: %w[id],
      message_id: %w[context_message_id]
    },
    indexed: {
      source_id: %w[id],
      message_id: %w[context_message_id]
    }
  },
  "images" => {
    required: {
      message_id: %w[context_message_id],
      image_id: %w[id]
    },
    indexed: {
      message_id: %w[context_message_id],
      image_id: %w[id]
    }
  },
  "uploaded_images" => {
    required: {
      message_id: %w[context_message_id],
      image_id: %w[data_id_unnamed data_id]
    },
    indexed: {
      message_id: %w[context_message_id],
      image_id: %w[data_id_unnamed data_id]
    }
  },
  "attachments" => {
    required: {
      message_id: %w[context_message_id],
      attachment_id: %w[data_id id]
    },
    indexed: {
      message_id: %w[context_message_id],
      attachment_id: %w[data_id id]
    }
  },
  "files" => {
    required: {
      url: %w[url id]
    },
    indexed: {
      source_collection: %w[data_source_collection],
      source_message_id: %w[data_source_message_id]
    }
  },
  "users_data_user_badges_items" => {
    required: {
      badge_id: %w[badge_id]
    },
    indexed: {
      badge_id: %w[badge_id]
    },
    child_of: "users"
  }
}.freeze
MAX_LISTED_IDS = 
5000

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ InputValidator

Returns a new instance of InputValidator.



392
393
394
395
396
397
398
399
400
401
402
403
404
 
# File 'lib/structured_data_to_sql/json/profiles/khoros_api_export.rb', line 392

def initialize
  @winners = Hash.new { |hash, table| hash[table] = {} }
  @child_issues = Hash.new { |hash, table| hash[table] = [] }
  @warnings = []
  @diagnostics = []
  @run_context = nil
  @observed_user_columns = Set.new
  @user_richness = {}
  @quality_signals =
    Hash.new { |hash, table| hash[table] = Hash.new(0) }
  @quality_signal_rows = Hash.new { |hash, table| hash[table] = [] }
  @validated = false
end

Instance Attribute Details

#diagnostics ⇒ Object (readonly)

warnings are the one-line strings; diagnostics carry the same facts structured (lists, run scoping, actions) for the terminal blocks and the diagnostics report. One diagnostic per warning.



385
386
387
 
# File 'lib/structured_data_to_sql/json/profiles/khoros_api_export.rb', line 385

def diagnostics
  @diagnostics
end

#run_context ⇒ Object

Optional run scoping: { manifest: Json::ExporterManifest or nil, source_paths: { "errors" => path, "gaps" => path } }.



388
389
390
 
# File 'lib/structured_data_to_sql/json/profiles/khoros_api_export.rb', line 388

def run_context
  @run_context
end

#warnings ⇒ Object (readonly)

warnings are the one-line strings; diagnostics carry the same facts structured (lists, run scoping, actions) for the terminal blocks and the diagnostics report. One diagnostic per warning.



385
386
387
 
# File 'lib/structured_data_to_sql/json/profiles/khoros_api_export.rb', line 385

def warnings
  @warnings
end

Instance Method Details

#observe(table, columns, sid, parent_sid: nil, context_columns: []) ⇒ Object

context_columns must be pre-sorted; the emitter computes the sorted list once per table definition.



408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
 
# File 'lib/structured_data_to_sql/json/profiles/khoros_api_export.rb', line 408

def observe(table, columns, sid, parent_sid: nil, context_columns: [])
  if QUALITY_SIGNAL_TABLES.key?(table)
    tally_quality_signal(table, columns)
  end
  spec = DIRECT[table] || OPTIONAL[table]
  return if spec.nil?
  if spec.key?(:child_of)
    observe_child(table, spec, columns, sid, parent_sid)
    return
  end

  winners = @winners[table]
  identity = identity_digest(columns, context_columns)
  if table == "users"
    @observed_user_columns.merge(columns.keys)
    observe_user_richness(identity, columns, sid)
  end
  previous = winners[identity]
  return if previous && previous[0] > sid

  winners[identity] = if DIRECT.key?(table)
    examine_direct(table, spec, columns, sid)
  else
    examine_optional(table, spec, columns, sid)
  end
end

#validate! ⇒ Object 



435
436
437
438
439
440
441
442
443
444
 
# File 'lib/structured_data_to_sql/json/profiles/khoros_api_export.rb', line 435

def validate!
  return if @validated

  DIRECT.each_key { |table| validate_direct(table) }
  OPTIONAL.each { |table, spec| validate_optional(table, spec) }
  validate_user_columns
  warn_credential_degraded_users
  report_quality_signals
  @validated = true
end