Class: Fluent::LmOutput

Inherits:
BufferedOutput
  • Object
show all
Defined in:
lib/fluent/plugin/out_lm.rb

Constant Summary collapse

RESOURCE_MAPPING_KEY = 
"_lm.resourceId".freeze
DEVICELESS_KEY_SERVICE = 
"resource.service.name".freeze
DEVICELESS_KEY_NAMESPACE = 
"resource.service.namespace".freeze

Instance Method Summary collapse

Instance Method Details

#configure(conf) ⇒ Object

This method is called before starting. ‘conf’ is a Hash that includes configuration parameters. If the configuration is invalid, raise Fluent::ConfigError.



58
59
60
 
# File 'lib/fluent/plugin/out_lm.rb', line 58

def configure(conf)
  super
end

#configure_auth ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/fluent/plugin/out_lm.rb', line 84

def configure_auth
  @use_bearer_instead_of_lmv1 = false
  if is_blank(@access_id) || is_blank(@access_key)
    log.info "Access Id or access key blank / null. Using bearer token for authentication."
    @use_bearer_instead_of_lmv1 = true
  end
  if @use_bearer_instead_of_lmv1 && is_blank(@bearer_token)
    log.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
    raise ArgumentError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
  end
end

#encode_if_necessary(str) ⇒ Object 



187
188
189
190
191
192
193
 
# File 'lib/fluent/plugin/out_lm.rb', line 187

def encode_if_necessary(str)
  if @force_encoding != ""
    return get_encoded_string(str)
  else
    return str
  end
end

#format(tag, time, record) ⇒ Object

This method is called when an event reaches to Fluentd. Convert the event to a raw string.



104
105
106
 
# File 'lib/fluent/plugin/out_lm.rb', line 104

def format(tag, time, record)
  [tag, time, record].to_msgpack
end

#generate_token(events) ⇒ Object 



231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'lib/fluent/plugin/out_lm.rb', line 231

def generate_token(events)

  if @use_bearer_instead_of_lmv1
    return "Bearer #{@bearer_token}"
  else
    timestamp = DateTime.now.strftime('%Q')
    signature = Base64.strict_encode64(
        OpenSSL::HMAC.hexdigest(
            OpenSSL::Digest.new('sha256'),
            @access_key,
            "POST#{timestamp}#{events.to_json}/log/ingest"
        )
    )
    return "LMv1 #{@access_id}:#{signature}:#{timestamp}"
  end
end

#get_encoded_string(str) ⇒ Object 



195
196
197
 
# File 'lib/fluent/plugin/out_lm.rb', line 195

def get_encoded_string(str)
  return str.force_encoding(@force_encoding).encode("UTF-8")
end

#get_metadata(record) ⇒ Object 



173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/fluent/plugin/out_lm.rb', line 173

def get_metadata(record)
  #if encoding is not defined we will skip going through each key val
  #and return the whole record for performance reasons in case of a bulky record.
  if @force_encoding == ""
    return record
  else
    lm_event = {}
    record.each do |key, value|
      lm_event["#{key}"] = get_encoded_string(value)
    end
    return lm_event
  end
end

#is_blank(str) ⇒ Object 



248
249
250
251
252
253
254
 
# File 'lib/fluent/plugin/out_lm.rb', line 248

def is_blank(str)
  if str.nil? || str.to_s.strip.empty?
    return true
  else
    return false
  end
end

#multi_workers_ready? ⇒ Boolean

Returns:

  • (Boolean) 


62
63
64
 
# File 'lib/fluent/plugin/out_lm.rb', line 62

def multi_workers_ready?
  true
end

#process_record(tag, time, record) ⇒ Object 



126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/fluent/plugin/out_lm.rb', line 126

def process_record(tag, time, record)
  resource_map = {}
  lm_event = {}

  if @include_metadata
    lm_event = get_metadata(record)
  end

  if !@device_less_logs
    # With devices
    if record[RESOURCE_MAPPING_KEY] == nil
        @resource_mapping.each do |key, value|
          k = value
          nestedVal = record
          key.to_s.split('.').each { |x| nestedVal = nestedVal[x] }
          if nestedVal != nil
            resource_map[k] = nestedVal
          end
        end
      lm_event[RESOURCE_MAPPING_KEY] = resource_map
    else
      lm_event[RESOURCE_MAPPING_KEY] = record[RESOURCE_MAPPING_KEY]
    end
  else
    # Device less
    if record[DEVICELESS_KEY_SERVICE]==nil
      log.error "When device_less_logs is set \'true\', record must have \'service\'. Ignoring this event #{lm_event}."
      return nil
    else
      lm_event[DEVICELESS_KEY_SERVICE] = encode_if_necessary(record[DEVICELESS_KEY_SERVICE])
      if record[DEVICELESS_KEY_NAMESPACE]!=nil
        lm_event[DEVICELESS_KEY_NAMESPACE] = encode_if_necessary(record[DEVICELESS_KEY_NAMESPACE])
      end
    end
  end

  if record["timestamp"] != nil
    lm_event["timestamp"] = record["timestamp"]
  else
    lm_event["timestamp"] = Time.at(time).utc.to_datetime.rfc3339
  end

  lm_event["message"] = encode_if_necessary(record["message"])

  return lm_event
end

#send_batch(events) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
 
# File 'lib/fluent/plugin/out_lm.rb', line 199

def send_batch(events)
  body = events.to_json

  if @debug
    log.info "Sending #{events.length} events to logic monitor at #{@url}"
    log.info "Request json #{body}"
  end

  request = Net::HTTP::Post.new(@uri.request_uri)
  request['authorization'] = generate_token(events)

  if @compression == "gzip"
    request['Content-Encoding'] = "gzip"
    gzip = Zlib::GzipWriter.new(StringIO.new)
    gzip << body
    request.body = gzip.close.string
  else
    request.body = body
  end

  if @debug
    log.info "Sending the below request headers to logicmonitor:"
    request.each_header {|key,value| log.info "#{key} = #{value}" }
  end

  resp = @http_client.request @uri, request
  if @debug || resp.kind_of?(Net::HTTPMultiStatus) || !resp.kind_of?(Net::HTTPSuccess)
    log.info "Status code:#{resp.code} Request Id:#{resp.header['x-request-id']} message:#{resp.body}"
  end
end

#shutdown ⇒ Object

This method is called when shutting down. Shutdown the thread and close sockets or files here.



97
98
99
100
 
# File 'lib/fluent/plugin/out_lm.rb', line 97

def shutdown
  super
  @http_client.shutdown
end

#start ⇒ Object

This method is called when starting. Open sockets or files here.



68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/fluent/plugin/out_lm.rb', line 68

def start
  super
  configure_auth
  proxy_uri = :ENV
  if @http_proxy
    proxy_uri = URI.parse(http_proxy)
  elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
    log.info("Using HTTP proxy defined in environment variable")
  end
  @http_client = Net::HTTP::Persistent.new name: "fluent-plugin-lm-logs", proxy: proxy_uri
  @http_client.override_headers["Content-Type"] = "application/json"
  @http_client.override_headers["User-Agent"] = log_source + "/" + LmLogsFluentPlugin::VERSION
  @url = "https://#{@company_name}.logicmonitor.com/rest/log/ingest"
  @uri = URI.parse(@url)
end

#write(chunk) ⇒ Object

This method is called every flush interval. Write the buffer chunk to files or databases here. ‘chunk’ is a buffer chunk that includes multiple formatted events. You can use ‘data = chunk.read’ to get all events and ‘chunk.open {|io| … }’ to get IO objects.

NOTE! This method is called by internal thread, not Fluentd’s main thread. So IO wait doesn’t affect other plugins.



115
116
117
118
119
120
121
122
123
124
 
# File 'lib/fluent/plugin/out_lm.rb', line 115

def write(chunk)
  events = []
  chunk.msgpack_each do |(tag, time, record)|
    event = process_record(tag,time,record)
    if event != nil
      events.push(event)
    end
  end
  send_batch(events)
end