Class: Peruby::Lexer

Inherits:
Object
  • Object
show all
Includes:
SourceScanner, StructureScanner, TermScanner
Defined in:
lib/peruby/lexer.rb,
lib/peruby/lexer/state.rb,
lib/peruby/lexer/token.rb,
lib/peruby/lexer/number.rb,
lib/peruby/lexer/heredoc.rb,
lib/peruby/lexer/keywords.rb,
lib/peruby/lexer/quote_like.rb,
lib/peruby/lexer/term_scanner.rb,
lib/peruby/lexer/source_scanner.rb,
lib/peruby/lexer/structure_scanner.rb

Overview

Stateful Perl lexer. It resolves term/operator ambiguities without parser feedback.

Defined Under Namespace

Modules: Keywords, Number, SourceScanner, StructureScanner, TermScanner Classes: Heredoc, Quote, QuoteLike, State, Token

Constant Summary collapse

OPERATORS = 
{
  '**=' => :ASSIGNOP, '//=' => :ASSIGNOP, '&&=' => :ASSIGNOP, '||=' => :ASSIGNOP,
  '<<=' => :ASSIGNOP, '>>=' => :ASSIGNOP, '&=' => :ASSIGNOP, '|=' => :ASSIGNOP, '^=' => :ASSIGNOP,
  '...' => :DOTDOTDOT, '**' => :POW,
  '//' => :DORDOR, '&&' => :ANDAND, '||' => :OROR, '=~' => :MATCHOP,
  '!~' => :NOTMATCHOP, '==' => :EQ, '!=' => :NE, '<=' => :LE, '>=' => :GE,
  '<=>' => :CMP, '->' => :ARROW, '=>' => :FATCOMMA, '..' => :DOTDOT,
  '<<' => :LSHIFT, '>>' => :RSHIFT, '++' => :INC, '--' => :DEC,
  '+=' => :ASSIGNOP, '-=' => :ASSIGNOP, '*=' => :ASSIGNOP, '/=' => :ASSIGNOP,
  '%=' => :ASSIGNOP, '.=' => :ASSIGNOP, 'x=' => :ASSIGNOP,
  '=' => :'=', '+' => :+, '-' => :-, '*' => :*, '/' => :/, '%' => :%, '.' => :'.',
  '<' => :<, '>' => :>, '!' => :!, '~' => :~, '&' => :&, '|' => :|, '^' => :^,
  '?' => :'?', ':' => :':', ',' => :',', ';' => :';', '\\' => :REFGEN
}.freeze
SORTED_OPERATORS = 
OPERATORS.keys.sort_by(&:length).reverse.freeze
SIGILS = 
{ '$' => :SCALAR_VAR, '@' => :ARRAY_VAR, '%' => :HASH_VAR,
'&' => :CODE_VAR, '*' => :GLOB_VAR }.freeze
QUOTE_OPERATORS = 
%w[q qq qw qr m s tr y].freeze
FILETEST = 
/\A-[erwxofdszlpSbcugktTBAMC](?!\w)/

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, file: '-e', known_subs: {}, implicit_semicolon: false) ⇒ Lexer

Returns a new instance of Lexer.



42
43
44
45
46
47
48
49
50
51
52
53
54
 
# File 'lib/peruby/lexer.rb', line 42

def initialize(source, file: '-e', known_subs: {}, implicit_semicolon: false)
  @source = source
  @file = file
  @known_subs = known_subs
  @implicit_semicolon_enabled = implicit_semicolon
  @index = 0
  @line = 1
  @column = 1
  @state = State.new
  @pending_heredocs = []
  @indirect_print = false
  @data = nil
end

Instance Attribute Details

#data ⇒ Object (readonly)

Returns the value of attribute data.



40
41
42
 
# File 'lib/peruby/lexer.rb', line 40

def data
  @data
end

#state ⇒ Object (readonly)

Returns the value of attribute state.



40
41
42
 
# File 'lib/peruby/lexer.rb', line 40

def state
  @state
end

Instance Method Details

#each ⇒ Object 



60
61
62
63
64
65
66
 
# File 'lib/peruby/lexer.rb', line 60

def each
  return enum_for(__method__) unless block_given?

  while (token = next_token)
    yield token
  end
end

#next_token ⇒ Object 



68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/peruby/lexer.rb', line 68

def next_token
  skip_ignored
  return eof if end?
  return implicit_block_semicolon if implicit_block_semicolon?

  line = @line
  column = @column
  type, value = scan_token
  @last_token_type = type
  Token.new(type, value, @file, line, column)
end

#tokens ⇒ Object 



56
57
58
 
# File 'lib/peruby/lexer.rb', line 56

def tokens
  each.to_a
end