Class: Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/parade_db/tokenizer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, positional_args, options) ⇒ Tokenizer

Returns a new instance of Tokenizer.



4
5
6
7
8
 
# File 'lib/parade_db/tokenizer.rb', line 4

def initialize(name, positional_args, options)
  @name = name
  @positional_args = positional_args
  @options = options
end

Instance Attribute Details

#name ⇒ Object (readonly)

Returns the value of attribute name.



2
3
4
 
# File 'lib/parade_db/tokenizer.rb', line 2

def name
  @name
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



2
3
4
 
# File 'lib/parade_db/tokenizer.rb', line 2

def options
  @options
end

#positional_args ⇒ Object (readonly)

Returns the value of attribute positional_args.



2
3
4
 
# File 'lib/parade_db/tokenizer.rb', line 2

def positional_args
  @positional_args
end

Class Method Details

.chinese_compatible(options: nil) ⇒ Object 



58
59
60
 
# File 'lib/parade_db/tokenizer.rb', line 58

def self.chinese_compatible(options: nil)
  new("chinese_compatible", nil, options)
end

.edge_ngram(min_gram, max_gram, options: nil) ⇒ Object 



50
51
52
 
# File 'lib/parade_db/tokenizer.rb', line 50

def self.edge_ngram(min_gram, max_gram, options: nil)
  new("edge_ngram", [min_gram, max_gram], options)
end

.icu(options: nil) ⇒ Object 



66
67
68
 
# File 'lib/parade_db/tokenizer.rb', line 66

def self.icu(options: nil)
  new("icu", nil, options)
end

.jieba(options: nil) ⇒ Object 



70
71
72
 
# File 'lib/parade_db/tokenizer.rb', line 70

def self.jieba(options: nil)
  new("jieba", nil, options)
end

.lindera(dictionary, options: nil) ⇒ Object 



62
63
64
 
# File 'lib/parade_db/tokenizer.rb', line 62

def self.lindera(dictionary, options: nil)
  new("lindera", [dictionary], options)
end

.literal(options: nil) ⇒ Object 



42
43
44
 
# File 'lib/parade_db/tokenizer.rb', line 42

def self.literal(options: nil)
  new("literal", nil, options)
end

.literal_normalized(options: nil) ⇒ Object 



46
47
48
 
# File 'lib/parade_db/tokenizer.rb', line 46

def self.literal_normalized(options: nil)
  new("literal_normalized", nil, options)
end

.ngram(min_gram, max_gram, options: nil) ⇒ Object 



34
35
36
 
# File 'lib/parade_db/tokenizer.rb', line 34

def self.ngram(min_gram, max_gram, options: nil)
  new("ngram", [min_gram, max_gram], options)
end

.regex_pattern(pattern, options: nil) ⇒ Object 



54
55
56
 
# File 'lib/parade_db/tokenizer.rb', line 54

def self.regex_pattern(pattern, options: nil)
  new("regex_pattern", [pattern], options)
end

.simple(options: nil) ⇒ Object 



38
39
40
 
# File 'lib/parade_db/tokenizer.rb', line 38

def self.simple(options: nil)
  new("simple", nil, options)
end

.source_code(options: nil) ⇒ Object 



74
75
76
 
# File 'lib/parade_db/tokenizer.rb', line 74

def self.source_code(options: nil)
  new("source_code", nil, options)
end

.unicode_words(options: nil) ⇒ Object 



30
31
32
 
# File 'lib/parade_db/tokenizer.rb', line 30

def self.unicode_words(options: nil)
  new("unicode_words", nil, options)
end

.whitespace(options: nil) ⇒ Object 



26
27
28
 
# File 'lib/parade_db/tokenizer.rb', line 26

def self.whitespace(options: nil)
  new("whitespace", nil, options)
end

Instance Method Details

#render ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/parade_db/tokenizer.rb', line 10

def render()
  if options.nil? && positional_args.nil?
    return "pdb.#{name}"
  end

  args = []
  if !positional_args.nil?
    args.concat(positional_args.map { |x| render_positional_arg(x) })
  end
  if !options.nil?
    args.concat(options.map {|k, v| quote_term("#{k}=#{v}")})
  end

  return "pdb.#{name}(#{args.join(",")})"
end