Module: LexerKit::CLI::Commands::Lex

Defined in:
lib/lexer_kit/cli/commands.rb

Overview

Tokenize input

Class Method Summary collapse

Class Method Details

.output_json(tokens) ⇒ Object 



221
222
223
224
225
226
227
228
229
230
231
232
233
 
# File 'lib/lexer_kit/cli/commands.rb', line 221

def self.output_json(tokens)
  result = tokens.map do |t|
    {
      token: t[:name].to_s,
      text: t[:text],
      line: t[:line],
      col: t[:col],
      start: t[:start],
      len: t[:len]
    }
  end
  puts JSON.pretty_generate(result)
end

.output_simple(tokens) ⇒ Object 



235
236
237
238
239
 
# File 'lib/lexer_kit/cli/commands.rb', line 235

def self.output_simple(tokens)
  tokens.each do |t|
    puts "#{t[:name]} #{t[:text].inspect}"
  end
end

.output_table(tokens, color) ⇒ Object 



241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
 
# File 'lib/lexer_kit/cli/commands.rb', line 241

def self.output_table(tokens, color)
  # Header
  puts "  LINE:COL  TOKEN            TEXT"
  puts "  --------  ---------------  --------------------"

  tokens.each do |t|
    pos = "#{t[:line]}:#{t[:col]}"
    name = t[:name].to_s
    text = truncate(t[:text].inspect, 40)

    if color
      puts "  #{pos.rjust(8)}  \e[36m#{name.ljust(15)}\e[0m  #{text}"
    else
      puts "  #{pos.rjust(8)}  #{name.ljust(15)}  #{text}"
    end
  end

  puts
  puts "#{tokens.size} tokens"
end

.run(argv) ⇒ Object 



152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
 
# File 'lib/lexer_kit/cli/commands.rb', line 152

def self.run(argv)
  options = { format: "table", color: $stdout.tty? }

  parser = OptionParser.new do |opts|
    opts.banner = "Usage: lexer_kit lex <lexer> [input]"

    opts.on("-f", "--format FORMAT", %w[table json simple], "Output format (table, json, simple)") do |v|
      options[:format] = v
    end

    opts.on("--no-color", "Disable colored output") do
      options[:color] = false
    end

    opts.on("-h", "--help", "Show this help") do
      puts opts
      return 0
    end
  end

  parser.parse!(argv)

  if argv.empty?
    warn "error: No lexer file specified"
    warn parser.banner
    return 1
  end

  lexer_path = argv.shift
  lexer = CLI.load_lexer(lexer_path)

  # Read input
  input = if argv.empty?
            $stdin.read
          else
            File.read(argv.shift)
          end

  # Collect tokens
  tokens = []
  source = Core::Source.new(input)
  source.line_index!

  lexer.lowlevel_each(input) do |tok_id, start, len|
    line, col = source.line_col(start)
    tokens << {
      id: tok_id,
      name: lexer.token_name(tok_id),
      text: input.byteslice(start, len),
      line: line,
      col: col,
      start: start,
      len: len
    }
  end

  # Output
  case options[:format]
  when "json"
    output_json(tokens)
  when "simple"
    output_simple(tokens)
  else
    output_table(tokens, options[:color])
  end

  0
end

.truncate(str, max) ⇒ Object 



262
263
264
265
266
267
268
 
# File 'lib/lexer_kit/cli/commands.rb', line 262

def self.truncate(str, max)
  if str.length > max
    "#{str[0, max - 3]}..."
  else
    str
  end
end