Class: Scrapetor::XPath::Parser

Inherits:
Object
  • Object
show all
Defined in:
lib/scrapetor/xpath.rb

Overview

Parser → AST

AST node shapes (all Hashes):

{ t: :path,  steps: [step,...], absolute: bool, double_slash: bool }
step: { axis: :child|:descendant_or_self|..., nt: nodetest, preds: [Expr,...] }
nodetest: :any_element | :text | :comment | :node | :pi(name=...) | {name: 'tag'} | :attr({name})
{ t: :or,  l:, r: }
{ t: :and, l:, r: }
{ t: :cmp, op: :eq|:neq|:lt|:le|:gt|:ge, l:, r: }
{ t: :add, op: :plus|:minus, l:, r: }
{ t: :mul, op: :mul|:div|:mod, l:, r: }
{ t: :neg, e: }
{ t: :union, ops: [Expr,...] }
{ t: :filter, primary: Expr, preds: [Expr,...] } chained with /path
{ t: :func, name: 'count', args: [Expr,...] }
{ t: :num, v: }
{ t: :str, v: }

Constant Summary collapse

AXIS_SYMBOLS = 
{
  "child"              => :child,
  "descendant"         => :descendant,
  "descendant-or-self" => :descendant_or_self,
  "parent"             => :parent,
  "self"               => :self,
  "ancestor"           => :ancestor,
  "ancestor-or-self"   => :ancestor_or_self,
  "following-sibling"  => :following_sibling,
  "preceding-sibling"  => :preceding_sibling,
  "following"          => :following,
  "preceding"          => :preceding,
  "attribute"          => :attribute,
  "namespace"          => :namespace
}.freeze

Instance Method Summary collapse

Constructor Details

#initialize(tokens, raw) ⇒ Parser

Returns a new instance of Parser.



241
242
243
244
245
 
# File 'lib/scrapetor/xpath.rb', line 241

def initialize(tokens, raw)
  @tokens = tokens
  @pos = 0
  @raw = raw
end

Instance Method Details

#consume ⇒ Object 



248
 
# File 'lib/scrapetor/xpath.rb', line 248

def consume; t = @tokens[@pos]; @pos += 1; t; end

#consume_path_separator(steps) ⇒ Object 



419
420
421
422
423
424
 
# File 'lib/scrapetor/xpath.rb', line 419

def consume_path_separator(steps)
  tk = consume[0]
  if tk == :slash_slash
    steps << { axis: :descendant_or_self, nt: :node, preds: [] }
  end
end

#expect(type) ⇒ Object 



249
250
251
252
253
254
255
256
 
# File 'lib/scrapetor/xpath.rb', line 249

def expect(type)
  t = @tokens[@pos]
  unless t && t[0] == type
    raise ParseError, "expected #{type} got #{t.inspect} in `#{@raw}`"
  end
  @pos += 1
  t
end

#location_path_start? ⇒ Boolean

Returns:

  • (Boolean) 


368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
 
# File 'lib/scrapetor/xpath.rb', line 368

def location_path_start?
  tk = peek[0]
  return true if %i[slash slash_slash dot dot_dot at star].include?(tk)
  # name token followed by something that looks like a node-test
  # (axis_sep, paren-without-args-as-function, slash, predicate)
  if tk == :name
    n2 = peek(1)[0]
    # `name::` is an axis
    return true if n2 == :axis_sep
    # `name(...)` -> function or nodetype()
    if n2 == :lparen
      # Distinguish nodetype() vs function call. Nodetypes are:
      # text, comment, node, processing-instruction.
      nm = peek[1]
      return %w[text comment node processing-instruction].include?(nm)
    end
    # bare name like `div` — that's a location step (child::div)
    return true
  end
  false
end

#parse_additive ⇒ Object 



298
299
300
301
302
303
304
305
 
# File 'lib/scrapetor/xpath.rb', line 298

def parse_additive
  l = parse_multiplicative
  while %i[plus minus].include?(peek[0])
    op = consume[0] == :plus ? :plus : :minus
    l = { t: :add, op: op, l: l, r: parse_multiplicative }
  end
  l
end

#parse_and ⇒ Object 



271
272
273
274
275
276
277
278
 
# File 'lib/scrapetor/xpath.rb', line 271

def parse_and
  l = parse_equality
  while peek[0] == :and_op
    consume
    l = { t: :and, l: l, r: parse_equality }
  end
  l
end

#parse_equality ⇒ Object 



280
281
282
283
284
285
286
287
 
# File 'lib/scrapetor/xpath.rb', line 280

def parse_equality
  l = parse_relational
  while %i[eq neq].include?(peek[0])
    op = consume[0]
    l = { t: :cmp, op: op, l: l, r: parse_relational }
  end
  l
end

#parse_expr ⇒ Object 



258
259
260
 
# File 'lib/scrapetor/xpath.rb', line 258

def parse_expr
  parse_or
end

#parse_location_path ⇒ Object 



390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
 
# File 'lib/scrapetor/xpath.rb', line 390

def parse_location_path
  steps = []
  absolute = false
  double = false
  if peek[0] == :slash_slash
    absolute = true; double = true
    steps << { axis: :descendant_or_self, nt: :node, preds: [] }
    consume
  elsif peek[0] == :slash
    absolute = true
    consume
    # If the next token doesn't start a step, this is just '/'.
    if !step_start?
      return { t: :path, steps: steps, absolute: absolute, double_slash: false }
    end
  end
  parse_relative_location_path_into(steps)
  { t: :path, steps: steps, absolute: absolute, double_slash: double }
end

#parse_multiplicative ⇒ Object 



307
308
309
310
311
312
313
314
315
316
317
 
# File 'lib/scrapetor/xpath.rb', line 307

def parse_multiplicative
  l = parse_unary
  loop do
    tk = peek[0]
    break unless %i[star_mul div_op mod_op].include?(tk)
    consume
    op = tk == :star_mul ? :mul : (tk == :div_op ? :div : :mod)
    l = { t: :mul, op: op, l: l, r: parse_unary }
  end
  l
end

#parse_node_test ⇒ Object

Raises:



478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
 
# File 'lib/scrapetor/xpath.rb', line 478

def parse_node_test
  if peek[0] == :star
    consume
    return :any_element
  end
  if peek[0] == :name && peek(1)[0] == :lparen
    nm = consume[1]
    expect(:lparen)
    case nm
    when "node"
      expect(:rparen)
      return :node
    when "text"
      expect(:rparen)
      return :text
    when "comment"
      expect(:rparen)
      return :comment
    when "processing-instruction"
      arg = nil
      if peek[0] == :string
        arg = consume[1]
      end
      expect(:rparen)
      return { pi: arg }
    else
      raise ParseError, "unexpected `#{nm}(` as node test in `#{@raw}`"
    end
  end
  if peek[0] == :name
    # tag name. Support qname (prefix:local) — we ignore the prefix.
    name = consume[1]
    if peek[0] == :colon && peek(1)[0] == :name
      consume
      name = consume[1]
    end
    return { name: name }
  end
  raise ParseError, "expected node test, got #{peek.inspect} in `#{@raw}`"
end

#parse_or ⇒ Object 



262
263
264
265
266
267
268
269
 
# File 'lib/scrapetor/xpath.rb', line 262

def parse_or
  l = parse_and
  while peek[0] == :or_op
    consume
    l = { t: :or, l: l, r: parse_and }
  end
  l
end

#parse_path ⇒ Object

PathExpr := LocationPath | FilterExpr (‘/’ RelativeLocationPath | ‘//’ RelativeLocationPath)?



343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
 
# File 'lib/scrapetor/xpath.rb', line 343

def parse_path
  if location_path_start?
    parse_location_path
  else
    primary = parse_primary
    # PrimaryExpr Predicate*
    preds = []
    while peek[0] == :lbracket
      consume
      preds << parse_expr
      expect(:rbracket)
    end
    base = preds.empty? ? primary : { t: :filter, primary: primary, preds: preds }
    # Optional /RelativeLocationPath or //RelativeLocationPath
    if %i[slash slash_slash].include?(peek[0])
      steps = []
      consume_path_separator(steps)
      parse_relative_location_path_into(steps)
      { t: :filter_path, primary: base, steps: steps }
    else
      base
    end
  end
end

#parse_primary ⇒ Object 



519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
 
# File 'lib/scrapetor/xpath.rb', line 519

def parse_primary
  tk = peek[0]
  case tk
  when :string
    { t: :str, v: consume[1] }
  when :number
    { t: :num, v: consume[1] }
  when :lparen
    consume
    e = parse_expr
    expect(:rparen)
    e
  when :name
    # FunctionCall
    fname = consume[1]
    expect(:lparen)
    args = []
    unless peek[0] == :rparen
      args << parse_expr
      while peek[0] == :comma
        consume
        args << parse_expr
      end
    end
    expect(:rparen)
    { t: :func, name: fname, args: args }
  else
    raise ParseError, "unexpected `#{peek.inspect}` in `#{@raw}`"
  end
end

#parse_relational ⇒ Object 



289
290
291
292
293
294
295
296
 
# File 'lib/scrapetor/xpath.rb', line 289

def parse_relational
  l = parse_additive
  while %i[lt le gt ge].include?(peek[0])
    op = consume[0]
    l = { t: :cmp, op: op, l: l, r: parse_additive }
  end
  l
end

#parse_relative_location_path_into(steps) ⇒ Object 



410
411
412
413
414
415
416
417
 
# File 'lib/scrapetor/xpath.rb', line 410

def parse_relative_location_path_into(steps)
  steps << parse_step
  loop do
    break unless %i[slash slash_slash].include?(peek[0])
    consume_path_separator(steps)
    steps << parse_step
  end
end

#parse_step ⇒ Object 



446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
 
# File 'lib/scrapetor/xpath.rb', line 446

def parse_step
  if peek[0] == :dot
    consume
    return { axis: :self, nt: :node, preds: [] }
  end
  if peek[0] == :dot_dot
    consume
    return { axis: :parent, nt: :node, preds: [] }
  end

  axis = :child
  if peek[0] == :at
    consume
    axis = :attribute
  elsif peek[0] == :name && peek(1)[0] == :axis_sep
    name = consume[1]
    axis_sym = AXIS_SYMBOLS[name]
    raise UnsupportedError, "unknown axis `#{name}` in `#{@raw}`" unless axis_sym
    axis = axis_sym
    expect(:axis_sep)
  end

  nt = parse_node_test
  preds = []
  while peek[0] == :lbracket
    consume
    preds << parse_expr
    expect(:rbracket)
  end
  { axis: axis, nt: nt, preds: preds }
end

#parse_unary ⇒ Object 



319
320
321
322
323
324
325
326
 
# File 'lib/scrapetor/xpath.rb', line 319

def parse_unary
  if peek[0] == :minus
    consume
    { t: :neg, e: parse_unary }
  else
    parse_union
  end
end

#parse_union ⇒ Object 



328
329
330
331
332
333
334
335
336
337
338
339
340
 
# File 'lib/scrapetor/xpath.rb', line 328

def parse_union
  l = parse_path
  if peek[0] == :pipe
    ops = [l]
    while peek[0] == :pipe
      consume
      ops << parse_path
    end
    { t: :union, ops: ops }
  else
    l
  end
end

#peek(k = 0) ⇒ Object 



247
 
# File 'lib/scrapetor/xpath.rb', line 247

def peek(k = 0); @tokens[@pos + k]; end

#step_start? ⇒ Boolean

Returns:

  • (Boolean) 


426
427
428
 
# File 'lib/scrapetor/xpath.rb', line 426

def step_start?
  %i[name star at dot dot_dot].include?(peek[0])
end