Module: Fbtxt

Defined in:
lib/fbtxt/lexer.rb,
lib/fbtxt/lexer.rb,
lib/fbtxt/lexer/lexer.rb,
lib/fbtxt/lexer/token.rb,
lib/fbtxt/lexer/version.rb,
lib/fbtxt/lexer/token-geo.rb,
lib/fbtxt/lexer/token-date.rb,
lib/fbtxt/lexer/token-note.rb,
lib/fbtxt/lexer/token-prop.rb,
lib/fbtxt/lexer/token-text.rb,
lib/fbtxt/lexer/token-time.rb,
lib/fbtxt/lexer/lexer-props.rb,
lib/fbtxt/lexer/lexer_token.rb,
lib/fbtxt/lexer/token-goals.rb,
lib/fbtxt/lexer/token-group.rb,
lib/fbtxt/lexer/token-round.rb,
lib/fbtxt/lexer/token-score.rb,
lib/fbtxt/lexer/lexer-logger.rb,
lib/fbtxt/lexer/lexer-on_top.rb,
lib/fbtxt/lexer/lexer_buffer.rb,
lib/fbtxt/lexer/token-status.rb,
lib/fbtxt/lexer/lexer-on_goal.rb,
lib/fbtxt/lexer/lexer_context.rb,
lib/fbtxt/lexer/lexer-prep_doc.rb,
lib/fbtxt/lexer/lexer-prep_line.rb,
lib/fbtxt/lexer/token-prop_name.rb,
lib/fbtxt/lexer/token-score_full.rb,
lib/fbtxt/lexer/token-score_legs.rb,
lib/fbtxt/lexer/token-date--names.rb,
lib/fbtxt/lexer/lexer-on_group_def.rb,
lib/fbtxt/lexer/lexer-on_prop_misc.rb,
lib/fbtxt/lexer/lexer-on_round_def.rb,
lib/fbtxt/lexer/token-score_fuller.rb,
lib/fbtxt/lexer/lexer-on_prop_cards.rb,
lib/fbtxt/lexer/lexer-tokenize_line.rb,
lib/fbtxt/lexer/lexer-tokenize_norm.rb,
lib/fbtxt/lexer/token-date--helpers.rb,
lib/fbtxt/lexer/token-date_duration.rb,
lib/fbtxt/lexer/token-status_inline.rb,
lib/fbtxt/lexer/lexer-on_prop_lineup.rb,
lib/fbtxt/lexer/token-goals--helpers.rb,
lib/fbtxt/lexer/token-score--helpers.rb,
lib/fbtxt/lexer/lexer-on_prop_penalties.rb

Overview

tokenize pipeline (step 2) - normalize tokens

transform (normalize) tokens (using simple patterns)
 to help along the (racc look ahead 1 - LA1) parser

Defined Under Namespace

Modules: Module Classes: Lexer, LexerResult, Tokens

Class Method Summary collapse

Class Method Details

.lex(txt, flatten: true) ⇒ Object

returns Fbtxt::LexerResult (ok?/tokens/errors)



108
109
110
111
112
113
 
# File 'lib/fbtxt/lexer.rb', line 108

def self.lex( txt, flatten: true )      ## returns Fbtxt::LexerResult (ok?/tokens/errors)
  lexer = Lexer.new( txt )
  ## move "wrapping" into result obj inside lexer - why? why not?
  tokens, errors = lexer.tokenize_with_errors( flatten: flatten )
  LexerResult.new( tokens, errors )
end