Class: Fluent::Plugin::CloudFoundrySyslogParser

Inherits:
Parser
  • Object
show all
Defined in:
lib/fluent/plugin/parser_cloudfoundry_syslog.rb

Constant Summary collapse

CF_IANA_ENTERPRISE_ID =

www.iana.org/assignments/enterprise-numbers/enterprise-numbers

47450.freeze
SYSLOG_HEADER_SPLIT_CHAR =

Syslog Constants datatracker.ietf.org/doc/html/rfc5424#section-6

" ".freeze
SYSLOG_NILVALUE =

See SP

"-".freeze
SYSLOG_PRI_DELIMITER_START =

See NILVALUE

"<".freeze
SYSLOG_PRI_DELIMITER_END =

See PRI

">".freeze
SYSLOG_SD_DELIMITER_START =

See PRI

"[".freeze
SYSLOG_SD_DELIMITER_END =

See STRUCTURED-DATA

"]".freeze
SYSLOG_HEADER_FIELDS =

datatracker.ietf.org/doc/html/rfc5424#section-6.2 PRI is parsed separately (datatracker.ietf.org/doc/html/rfc5424#section-6.2.1)

[
  "version",    # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.2
  "timestamp",  # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
  "hostname",   # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.4
  "app_name",   # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.5
  "proc_id",    # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.6
  "msg_id",     # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.7
]
SYSLOG_SEVERITY_CODES =

datatracker.ietf.org/doc/html/rfc5424#appendix-A.3 resources.docs.pivotal.io/pdfs/tiledev-guide-2.1.pdf

[
  "emergency",
  "alert",
  "critical",
  "error",
  "warning",
  "notice",
  "info",
  "debug",
]
SYSLOG_STRUCTURED_DATA_MATCH_REGEX =

Regexes to extract information from STRUCTURED-DATA Matches a whole STRUCTURED-DATA block including the delimiters '[', ']'

Regexp.new(/^(\[(?:[a-zA-Z_-]+(?:\@[0-9]+)?)*(?:[a-zA-Z0-9_-]+="(?:[^\\\]\"]|\\"|\\\]|\\\\|\\[^"\]\\])*"| )*\])/)
SYSLOG_SD_ID_MATCH_REGEX =

Matches the SD-ID if applied to the SYSLOG_STRUCTURED_DATA_MATCH_REGEX match

Regexp.new(/^\[([a-zA-Z0-9_-]+(?:\@[0-9]+)?)/)
SYSLOG_SD_PARAM_MATCH_REGEX =

Matches an SD-PARAM (SD-NAME=SD-VALUE)

Regexp.new(/([a-zA-Z0-9_-]+="(?:[^\\\]\"]|\\"|\\\]|\\\\|\\[^"\]\\])*")/)
GOROUTER_MESSAGE_STATIC_REGEX =

Regex to extract information from Gorouter access logs github.com/cloudfoundry/gorouter#access-logs <Request Host> - [<Start Date>] “<Request Method> <Request URL> <Request Protocol>” <Status Code> <Bytes Received> <Bytes Sent> “<Referer>” “<User-Agent>” <Remote Address> <Backend Address> x_forwarded_for:“<X-Forwarded-For>” x_forwarded_proto:“<X-Forwarded-Proto>” vcap_request_id:<X-Vcap-Request-ID> response_time:<Response Time> gorouter_time:<Gorouter Time> app_id:<Application ID> app_index:<Application Index> x_cf_routererror:<X-Cf-RouterError> <Extra Headers>

Regexp.new(/^(?<host>[^ ]+) - \[(?<timestamp>[^\]]+)\] "(?<method>[^ ]+) (?<pathname>[^ ]+) (?<protocol>[^"]+)" (?<status>[^ ]+) (?<bytes_received>[^ ]+) (?<bytes_sent>[^ ]+) "(?<referer>[^"]+)" "(?<user_agent>[^"]+)" "(?<remote_address>[^"]+)" "(?<backend_address>[^"]+)"/)
GOROUTER_MESSAGE_EXTRADATA_REGEX = 
Regexp.new(/(?<param>[a-zA-Z0-9_-]+):(?<value>(?:[0-9\.]+|"[^"]+"|-))(?: |\n|\\|$)/)

Instance Method Summary collapse

Constructor Details

#initialize ⇒ CloudFoundrySyslogParser

Returns a new instance of CloudFoundrySyslogParser.



63
64
65
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 63

def initialize
  super
end

Instance Method Details

#configure(conf) ⇒ Object 



67
68
69
70
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 67

def configure(conf)
  super
  @time_parser = time_parser_create(format: "%Y-%m-%dT%H:%M:%S.%L%z")
end

#parse(text) {|time, record| ... } ⇒ Object

Yields:

  • (time, record) 


72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 72

def parse(text)
  if text.nil? or not text.start_with?(SYSLOG_PRI_DELIMITER_START)
    yield nil
    return
  end

  cursor = 0
  record = {}

  if @include_raw_message
    record["raw"] = text
  end

  # RFC 5424 currently only defines version 1
  # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.2
  record["header"], cursor = parse_header(text)
  if cursor.nil? or record.dig("header", "version") != "1"
    yield nil
    return
  end

  # Convert to integer for convenience
  record["header"]["version"] = 1

  # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3
  time = @time_parser.parse(record["header"]["timestamp"]) rescue nil

  if time.nil?
    yield nil
    return
  end

  # Parse STRUCTURED_DATA
  record["sd"], cursor = parse_structured_data(text, cursor)
  if (cursor.nil?)
    yield nil
    return
  end

  # Parse MESSAGE
  msg = text.slice(cursor, text.length - cursor)

  if msg.nil?
    record["message"] = nil
  else
    record["message"] = msg.strip

    if @parse_gorouter_access_log and
       record.dig("sd", "tags@#{CF_IANA_ENTERPRISE_ID}", "origin") == "gorouter"
      record["gorouter"] = parse_gorouter_access_logs(record["message"])
    end
  end

  yield time, record
end

#parse_gorouter_access_logs(msg) ⇒ Object

github.com/cloudfoundry/gorouter#access-logs



214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 214

def parse_gorouter_access_logs(msg)
  r = msg.match(GOROUTER_MESSAGE_STATIC_REGEX)
  if r.nil? then return end
  extra_headers = msg[r.to_s.length..-1]
  r = r.named_captures
  if extra_headers.nil? then return r end
  extra_headers.strip.scan(GOROUTER_MESSAGE_EXTRADATA_REGEX) { |match|
    unless match.length == 2 then next end
    if match[1].start_with?('"') and match[1].end_with?('"')
      # Strings
      r[match[0]] = match[1][1..-2]
    elsif match[1].match(/^[0-9]+(?:\.[0-9+]+)?$/)
      # Numbers
      r[match[0]] = match[1].to_f
    else
      r[match[0]] = match[1]
    end
  }
  return r
end

#parse_header(text) ⇒ Object

datatracker.ietf.org/doc/html/rfc5424#section-6.2



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 151

def parse_header(text)
  facility, severity, c = parse_pri(text)
  if (c.nil?) then return end
  c = c + 1

  r = {
    "pri" => {
      "facility" => facility,
      "severity" => severity,
    },
  }

  SYSLOG_HEADER_FIELDS.each { |field|
    block, endIdx = parse_header_block(text, c)
    if block.nil? then return end
    r[field] = block
    c = endIdx
  }

  return r, c
end

#parse_header_block(text, startIdx) ⇒ Object 



134
135
136
137
138
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 134

def parse_header_block(text, startIdx)
  i = text.index(SYSLOG_HEADER_SPLIT_CHAR, startIdx)
  if i.nil? or i - startIdx < 1 then return end
  return text.slice(startIdx, i - startIdx), i + 1
end

#parse_integer(str) ⇒ Object 



128
129
130
131
132
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 128

def parse_integer(str)
  return Integer(str || "")
rescue ArgumentError
  return
end

#parse_pri(text) ⇒ Object

datatracker.ietf.org/doc/html/rfc5424#section-6.2.1



141
142
143
144
145
146
147
148
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 141

def parse_pri(text)
  unless text.start_with?(SYSLOG_PRI_DELIMITER_START) then return end
  endIdx = text.index(SYSLOG_PRI_DELIMITER_END, 1)
  if endIdx.nil? or endIdx < 2 then return end
  v_pri = parse_integer(text.slice(1, endIdx - 1))
  if v_pri.nil? or v_pri < 0 then return end
  return v_pri >> 3, SYSLOG_SEVERITY_CODES[v_pri & 0b111], endIdx
end

#parse_sd_element(sd_element) ⇒ Object

datatracker.ietf.org/doc/html/rfc5424#section-6.3.1



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 174

def parse_sd_element(sd_element)
  sd_params = {}

  # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.2
  sd_id = sd_element[SYSLOG_SD_ID_MATCH_REGEX]
  if sd_id.nil? then return end
  sd_id = sd_id[1..-1]

  # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.3
  sd_element.scan(SYSLOG_SD_PARAM_MATCH_REGEX).each { |match|
    arr = match[0].strip.split("=", 2)
    sd_params[arr[0]] = arr[1][1..-2]
  }

  return sd_id, sd_params
end

#parse_structured_data(text, startIdx) ⇒ Object

datatracker.ietf.org/doc/html/rfc5424#section-6.3



192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 192

def parse_structured_data(text, startIdx)
  if text[startIdx] == SYSLOG_NILVALUE then return {}, startIdx + 1 end
  unless text[startIdx] == SYSLOG_SD_DELIMITER_START then return end

  sd = text[startIdx..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX]
  if sd.nil? then return end

  r = {}
  len = 0
  loop do
    len += sd.length
    sd_id, sd_params = parse_sd_element(sd)
    if sd_id.nil? then return end
    r[sd_id] = sd_params
    sd = text[startIdx + len..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX]
    break if sd.nil?
  end

  return r, startIdx + len
end