Class: Traject::SolrPool::SolrJsonWriter

Inherits:
Object
  • Object
show all
Defined in:
lib/traject/solr_pool/solr_json_writer.rb

Overview

Drop-in replacement for Traject::SolrJsonWriter that sends every Solr HTTP call through a persistent http_connection_pool pool. Reuses the stock writer's settings vocabulary and public surface; all HTTP is delegated to Traject::SolrPool::Connection.

Defined Under Namespace

Classes: BadHttpResponse, MaxSkippedRecordsExceeded

Constant Summary collapse

URI_REGEXP = 
URI::DEFAULT_PARSER.make_regexp.freeze
DEFAULT_MAX_SKIPPED = 
0
DEFAULT_BATCH_SIZE = 
100

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(arg_settings) ⇒ SolrJsonWriter

Returns a new instance of SolrJsonWriter.



53
54
55
56
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 53

def initialize(arg_settings)
  @settings = Traject::Indexer::Settings.new(arg_settings)
  configure_from_settings
end

Instance Attribute Details

#batched_queue ⇒ Object (readonly)

Returns the value of attribute batched_queue.



50
51
52
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 50

def batched_queue
  @batched_queue
end

#connection ⇒ Object (readonly)

Returns the value of attribute connection.



50
51
52
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 50

def connection
  @connection
end

#pool_size ⇒ Object (readonly)

Returns the value of attribute pool_size.



50
51
52
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 50

def pool_size
  @pool_size
end

#settings ⇒ Object (readonly)

Returns the value of attribute settings.



50
51
52
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 50

def settings
  @settings
end

#solr_update_url ⇒ Object (readonly)

Returns the value of attribute solr_update_url.



50
51
52
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 50

def solr_update_url
  @solr_update_url
end

#thread_pool_size ⇒ Object (readonly)

Returns the value of attribute thread_pool_size.



50
51
52
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 50

def thread_pool_size
  @thread_pool_size
end

Instance Method Details

#close ⇒ Object 



111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 111

def close
  @thread_pool.raise_collected_exception!

  batch = Traject::Util.drain_queue(@batched_queue)
  @thread_pool.maybe_in_thread_pool { send_batch(batch) } unless batch.empty?

  shutdown_thread_pool if @thread_pool_size&.positive?

  @thread_pool.raise_collected_exception!
  commit if @commit_on_close
  # Pool is intentionally left warm in the registry for reuse by later
  # writers/readers on this origin; teardown is the host app's concern.
end

#commit(query_params = nil) ⇒ Object 



125
126
127
128
129
130
131
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 125

def commit(query_params = nil)
  query_params ||= @commit_solr_update_args || { 'commit' => 'true' }
  logger.info("#{self.class.name} sending commit to solr at url #{@solr_update_url}...")

  resp = connection.get(request_path_for(query_params), timeout: @commit_timeout)
  raise "Could not commit to Solr: #{resp.code} #{resp}" unless resp.status == 200
end

#delete(id) ⇒ Object 



133
134
135
136
137
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 133

def delete(id)
  json_package = JSON.generate(delete: id)
  resp = connection.post(request_path_for(@solr_update_args), body: json_package)
  raise "Could not delete #{id.inspect}, http response #{resp.code}: #{resp}" unless resp.status == 200
end

#delete_all! ⇒ Object 



139
140
141
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 139

def delete_all!
  delete(query: '*:*')
end

#flush ⇒ Object 



77
78
79
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 77

def flush
  send_batch(Traject::Util.drain_queue(@batched_queue))
end

#logger ⇒ Object 



64
65
66
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 64

def logger
  settings['logger'] ||= Yell.new($stderr, level: 'gt.fatal')
end

#put(context) ⇒ Object 



68
69
70
71
72
73
74
75
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 68

def put(context)
  @thread_pool.raise_collected_exception!
  @batched_queue << context
  return if @batched_queue.size < @batch_size

  batch = Traject::Util.drain_queue(@batched_queue)
  @thread_pool.maybe_in_thread_pool(batch) { |b| send_batch(b) }
end

#send_batch(batch) ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 81

def send_batch(batch)
  return if batch.empty?

  json_package = JSON.generate(batch.map(&:output_hash))
  begin
    resp = connection.post(request_path_for(@solr_update_args), body: json_package)
  rescue StandardError => e
    exception = e
  end

  return if exception.nil? && resp.status == 200

  log_batch_failure(exception, resp)
  batch.each { |c| send_single(c) }
end

#send_single(context) ⇒ Object 



97
98
99
100
101
102
103
104
105
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 97

def send_single(context)
  json_package = JSON.generate([context.output_hash])
  resp = connection.post(request_path_for(@solr_update_args), body: json_package)
  unless resp.status == 200
    raise BadHttpResponse.new("Unexpected HTTP response status #{resp.code} from POST", resp)
  end
rescue *skippable_exceptions => e
  handle_skipped(context, e)
end

#skipped_record_count ⇒ Object 



107
108
109
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 107

def skipped_record_count
  @skipped_record_incrementer.value
end

#solr_update_url_with_query(query_params) ⇒ Object 



58
59
60
61
62
 
# File 'lib/traject/solr_pool/solr_json_writer.rb', line 58

def solr_update_url_with_query(query_params)
  return @solr_update_url unless query_params

  "#{@solr_update_url}?#{URI.encode_www_form(query_params)}"
end