Class: Fluent::Plugin::CloudFoundrySyslogParser
- Inherits:
-
Parser
- Object
- Parser
- Fluent::Plugin::CloudFoundrySyslogParser
- Defined in:
- lib/fluent/plugin/parser_cloudfoundry_syslog.rb
Constant Summary collapse
- CF_IANA_ENTERPRISE_ID =
47450.freeze
- SYSLOG_HEADER_SPLIT_CHAR =
Syslog Constants datatracker.ietf.org/doc/html/rfc5424#section-6
" ".freeze
- SYSLOG_NILVALUE =
See SP
"-".freeze
- SYSLOG_PRI_DELIMITER_START =
See NILVALUE
"<".freeze
- SYSLOG_PRI_DELIMITER_END =
See PRI
">".freeze
- SYSLOG_SD_DELIMITER_START =
See PRI
"[".freeze
- SYSLOG_SD_DELIMITER_END =
See STRUCTURED-DATA
"]".freeze
- SYSLOG_HEADER_FIELDS =
datatracker.ietf.org/doc/html/rfc5424#section-6.2 PRI is parsed separately (datatracker.ietf.org/doc/html/rfc5424#section-6.2.1)
[ "version", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.2 "timestamp", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3 "hostname", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.4 "app_name", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.5 "proc_id", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.6 "msg_id", # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.7 ]
- SYSLOG_SEVERITY_CODES =
[ "emergency", "alert", "critical", "error", "warning", "notice", "info", "debug", ]
- SYSLOG_STRUCTURED_DATA_MATCH_REGEX =
Regexes to extract information from STRUCTURED-DATA Matches a whole STRUCTURED-DATA block including the delimiters '[', ']'
Regexp.new(/^(\[(?:[a-zA-Z_-]+(?:\@[0-9]+)?)*(?:[a-zA-Z0-9_-]+="(?:[^\\\]\"]|\\"|\\\]|\\\\|\\[^"\]\\])*"| )*\])/)
- SYSLOG_SD_ID_MATCH_REGEX =
Matches the SD-ID if applied to the SYSLOG_STRUCTURED_DATA_MATCH_REGEX match
Regexp.new(/^\[([a-zA-Z0-9_-]+(?:\@[0-9]+)?)/)
- SYSLOG_SD_PARAM_MATCH_REGEX =
Matches an SD-PARAM (SD-NAME=SD-VALUE)
Regexp.new(/([a-zA-Z0-9_-]+="(?:[^\\\]\"]|\\"|\\\]|\\\\|\\[^"\]\\])*")/)
- GOROUTER_MESSAGE_STATIC_REGEX =
Regex to extract information from Gorouter access logs github.com/cloudfoundry/gorouter#access-logs <Request Host> - [<Start Date>] “<Request Method> <Request URL> <Request Protocol>” <Status Code> <Bytes Received> <Bytes Sent> “<Referer>” “<User-Agent>” <Remote Address> <Backend Address> x_forwarded_for:“<X-Forwarded-For>” x_forwarded_proto:“<X-Forwarded-Proto>” vcap_request_id:<X-Vcap-Request-ID> response_time:<Response Time> gorouter_time:<Gorouter Time> app_id:<Application ID> app_index:<Application Index> x_cf_routererror:<X-Cf-RouterError> <Extra Headers>
Regexp.new(/^(?<host>[^ ]+) - \[(?<timestamp>[^\]]+)\] "(?<method>[^ ]+) (?<pathname>[^ ]+) (?<protocol>[^"]+)" (?<status>[^ ]+) (?<bytes_received>[^ ]+) (?<bytes_sent>[^ ]+) "(?<referer>[^"]+)" "(?<user_agent>[^"]+)" "(?<remote_address>[^"]+)" "(?<backend_address>[^"]+)"/)
- GOROUTER_MESSAGE_EXTRADATA_REGEX =
Regexp.new(/(?<param>[a-zA-Z0-9_-]+):(?<value>(?:[0-9\.]+|"[^"]+"|-))(?: |\n|\\|$)/)
Instance Method Summary collapse
- #configure(conf) ⇒ Object
-
#initialize ⇒ CloudFoundrySyslogParser
constructor
A new instance of CloudFoundrySyslogParser.
- #parse(text) {|time, record| ... } ⇒ Object
- #parse_gorouter_access_logs(msg) ⇒ Object
- #parse_header(text) ⇒ Object
- #parse_header_block(text, startIdx) ⇒ Object
- #parse_integer(str) ⇒ Object
- #parse_pri(text) ⇒ Object
- #parse_sd_element(sd_element) ⇒ Object
- #parse_structured_data(text, startIdx) ⇒ Object
Constructor Details
#initialize ⇒ CloudFoundrySyslogParser
Returns a new instance of CloudFoundrySyslogParser.
63 64 65 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 63 def initialize super end |
Instance Method Details
#configure(conf) ⇒ Object
67 68 69 70 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 67 def configure(conf) super @time_parser = time_parser_create(format: "%Y-%m-%dT%H:%M:%S.%L%z") end |
#parse(text) {|time, record| ... } ⇒ Object
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 72 def parse(text) if text.nil? or not text.start_with?(SYSLOG_PRI_DELIMITER_START) yield nil return end cursor = 0 record = {} if @include_raw_message record["raw"] = text end # RFC 5424 currently only defines version 1 # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.2 record["header"], cursor = parse_header(text) if cursor.nil? or record.dig("header", "version") != "1" yield nil return end # Convert to integer for convenience record["header"]["version"] = 1 # https://datatracker.ietf.org/doc/html/rfc5424#section-6.2.3 time = @time_parser.parse(record["header"]["timestamp"]) rescue nil if time.nil? yield nil return end # Parse STRUCTURED_DATA record["sd"], cursor = parse_structured_data(text, cursor) if (cursor.nil?) yield nil return end # Parse MESSAGE msg = text.slice(cursor, text.length - cursor) if msg.nil? record["message"] = nil else record["message"] = msg.strip if @parse_gorouter_access_log and record.dig("sd", "tags@#{CF_IANA_ENTERPRISE_ID}", "origin") == "gorouter" record["gorouter"] = parse_gorouter_access_logs(record["message"]) end end yield time, record end |
#parse_gorouter_access_logs(msg) ⇒ Object
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 214 def parse_gorouter_access_logs(msg) r = msg.match(GOROUTER_MESSAGE_STATIC_REGEX) if r.nil? then return end extra_headers = msg[r.to_s.length..-1] r = r.named_captures if extra_headers.nil? then return r end extra_headers.strip.scan(GOROUTER_MESSAGE_EXTRADATA_REGEX) { |match| unless match.length == 2 then next end if match[1].start_with?('"') and match[1].end_with?('"') # Strings r[match[0]] = match[1][1..-2] elsif match[1].match(/^[0-9]+(?:\.[0-9+]+)?$/) # Numbers r[match[0]] = match[1].to_f else r[match[0]] = match[1] end } return r end |
#parse_header(text) ⇒ Object
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 151 def parse_header(text) facility, severity, c = parse_pri(text) if (c.nil?) then return end c = c + 1 r = { "pri" => { "facility" => facility, "severity" => severity, }, } SYSLOG_HEADER_FIELDS.each { |field| block, endIdx = parse_header_block(text, c) if block.nil? then return end r[field] = block c = endIdx } return r, c end |
#parse_header_block(text, startIdx) ⇒ Object
134 135 136 137 138 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 134 def parse_header_block(text, startIdx) i = text.index(SYSLOG_HEADER_SPLIT_CHAR, startIdx) if i.nil? or i - startIdx < 1 then return end return text.slice(startIdx, i - startIdx), i + 1 end |
#parse_integer(str) ⇒ Object
128 129 130 131 132 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 128 def parse_integer(str) return Integer(str || "") rescue ArgumentError return end |
#parse_pri(text) ⇒ Object
141 142 143 144 145 146 147 148 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 141 def parse_pri(text) unless text.start_with?(SYSLOG_PRI_DELIMITER_START) then return end endIdx = text.index(SYSLOG_PRI_DELIMITER_END, 1) if endIdx.nil? or endIdx < 2 then return end v_pri = parse_integer(text.slice(1, endIdx - 1)) if v_pri.nil? or v_pri < 0 then return end return v_pri >> 3, SYSLOG_SEVERITY_CODES[v_pri & 0b111], endIdx end |
#parse_sd_element(sd_element) ⇒ Object
174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 174 def parse_sd_element(sd_element) sd_params = {} # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.2 sd_id = sd_element[SYSLOG_SD_ID_MATCH_REGEX] if sd_id.nil? then return end sd_id = sd_id[1..-1] # https://datatracker.ietf.org/doc/html/rfc5424#section-6.3.3 sd_element.scan(SYSLOG_SD_PARAM_MATCH_REGEX).each { |match| arr = match[0].strip.split("=", 2) sd_params[arr[0]] = arr[1][1..-2] } return sd_id, sd_params end |
#parse_structured_data(text, startIdx) ⇒ Object
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 |
# File 'lib/fluent/plugin/parser_cloudfoundry_syslog.rb', line 192 def parse_structured_data(text, startIdx) if text[startIdx] == SYSLOG_NILVALUE then return {}, startIdx + 1 end unless text[startIdx] == SYSLOG_SD_DELIMITER_START then return end sd = text[startIdx..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX] if sd.nil? then return end r = {} len = 0 loop do len += sd.length sd_id, sd_params = parse_sd_element(sd) if sd_id.nil? then return end r[sd_id] = sd_params sd = text[startIdx + len..-1][SYSLOG_STRUCTURED_DATA_MATCH_REGEX] break if sd.nil? end return r, startIdx + len end |