Class: Opencdd::Cddal::Lexer

Inherits:
Object
  • Object
show all
Defined in:
lib/opencdd/cddal/lexer.rb

Constant Summary collapse

PUNCTUATION = 
{
  "{" => :LBRACE, "}" => :RBRACE,
  ":" => :COLON,  "," => :COMMA,
  "<" => :LANGLE, ">" => :RANGLE,
  "." => :DOT,
  "(" => :LPAREN, ")" => :RPAREN,
}.freeze
KEYWORD_MAP = 
{
  "instance" => :INSTANCE,
  "alias"    => :ALIAS,
  "import"   => :IMPORT,
  "true"     => :TRUE,
  "false"    => :FALSE,
  "null"     => :NULL,
}.freeze
SOFT_KEYWORDS =

Soft keywords. These are reserved only inside import declarations; elsewhere they're valid identifiers (e.g. short_name.en: as for attosecond). The lexer emits IDENT and the parser validates the value in import position.

%w[as from].freeze
IRDI_RE = 
/[0-9]+\/[0-9]+\/\/\/[A-Za-z0-9_]+(?:_[0-9]+)?#[A-Za-z0-9_]+/.freeze
LOCAL_REF_RE = 
/[A-Za-z_][A-Za-z0-9_]*#[A-Za-z0-9_]+(?:##[A-Za-z0-9_]+)?/.freeze
DATE_RE = 
/\d{4}-\d{2}-\d{2}/.freeze
NUMBER_RE = 
/-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/.freeze
IDENT_RE = 
/[A-Za-z_][A-Za-z0-9_]*/.freeze
META_CLASS_RE = 
/(?:property-meta-class|enumeration-meta-class|term-meta-class|meta-class)(?![A-Za-z0-9_-])/.freeze
WS_RE = 
/[ \t\r]+/.freeze
NEWLINE_RE = 
/\n/.freeze
COMMENT_RE = 
/#[^\n]*/.freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source) ⇒ Lexer

Returns a new instance of Lexer.



49
50
51
52
53
54
 
# File 'lib/opencdd/cddal/lexer.rb', line 49

def initialize(source)
  @source = source.to_s
  @ss = StringScanner.new(@source)
  @line = 1
  @col = 1
end

Instance Attribute Details

#source ⇒ Object (readonly)

Returns the value of attribute source.



47
48
49
 
# File 'lib/opencdd/cddal/lexer.rb', line 47

def source
  @source
end

Instance Method Details

#next_token ⇒ Object 



70
71
72
73
74
 
# File 'lib/opencdd/cddal/lexer.rb', line 70

def next_token
  tok = tokens[@pos || 0]
  @pos = (@pos || 0) + 1
  tok ? [tok.kind, tok.value] : [false, false]
end

#tokens ⇒ Object 



56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/opencdd/cddal/lexer.rb', line 56

def tokens
  return @tokens if @tokens
  @tokens = []
  until @ss.eos?
    skip_interstitial
    break if @ss.eos?
    tok = scan_token
    raise LexError, "unexpected character #{@ss.peek(1).inspect} at line #{@line}, column #{@col}" unless tok
    @tokens << tok
  end
  @tokens << Token.new(kind: :EOF, value: "", line: @line, column: @col)
  @tokens
end