Class: ParadeDB::Tokenizer

Inherits:
Object
  • Object
show all
Defined in:
lib/parade_db/tokenizer.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(name, positional_args, options) ⇒ Tokenizer

Returns a new instance of Tokenizer.



5
6
7
8
9
 
# File 'lib/parade_db/tokenizer.rb', line 5

def initialize(name, positional_args, options)
  @name = name
  @positional_args = positional_args
  @options = options
end

Instance Attribute Details

#name ⇒ Object (readonly)

Returns the value of attribute name.



3
4
5
 
# File 'lib/parade_db/tokenizer.rb', line 3

def name
  @name
end

#options ⇒ Object (readonly)

Returns the value of attribute options.



3
4
5
 
# File 'lib/parade_db/tokenizer.rb', line 3

def options
  @options
end

#positional_args ⇒ Object (readonly)

Returns the value of attribute positional_args.



3
4
5
 
# File 'lib/parade_db/tokenizer.rb', line 3

def positional_args
  @positional_args
end

Class Method Details

.chinese_compatible(options: nil) ⇒ Object 



59
60
61
 
# File 'lib/parade_db/tokenizer.rb', line 59

def self.chinese_compatible(options: nil)
  new("chinese_compatible", nil, options)
end

.edge_ngram(min_gram, max_gram, options: nil) ⇒ Object 



51
52
53
 
# File 'lib/parade_db/tokenizer.rb', line 51

def self.edge_ngram(min_gram, max_gram, options: nil)
  new("edge_ngram", [min_gram, max_gram], options)
end

.icu(options: nil) ⇒ Object 



67
68
69
 
# File 'lib/parade_db/tokenizer.rb', line 67

def self.icu(options: nil)
  new("icu", nil, options)
end

.jieba(options: nil) ⇒ Object 



71
72
73
 
# File 'lib/parade_db/tokenizer.rb', line 71

def self.jieba(options: nil)
  new("jieba", nil, options)
end

.lindera(dictionary, options: nil) ⇒ Object 



63
64
65
 
# File 'lib/parade_db/tokenizer.rb', line 63

def self.lindera(dictionary, options: nil)
  new("lindera", [dictionary], options)
end

.literal(options: nil) ⇒ Object 



43
44
45
 
# File 'lib/parade_db/tokenizer.rb', line 43

def self.literal(options: nil)
  new("literal", nil, options)
end

.literal_normalized(options: nil) ⇒ Object 



47
48
49
 
# File 'lib/parade_db/tokenizer.rb', line 47

def self.literal_normalized(options: nil)
  new("literal_normalized", nil, options)
end

.ngram(min_gram, max_gram, options: nil) ⇒ Object 



35
36
37
 
# File 'lib/parade_db/tokenizer.rb', line 35

def self.ngram(min_gram, max_gram, options: nil)
  new("ngram", [min_gram, max_gram], options)
end

.regex_pattern(pattern, options: nil) ⇒ Object 



55
56
57
 
# File 'lib/parade_db/tokenizer.rb', line 55

def self.regex_pattern(pattern, options: nil)
  new("regex_pattern", [pattern], options)
end

.simple(options: nil) ⇒ Object 



39
40
41
 
# File 'lib/parade_db/tokenizer.rb', line 39

def self.simple(options: nil)
  new("simple", nil, options)
end

.source_code(options: nil) ⇒ Object 



75
76
77
 
# File 'lib/parade_db/tokenizer.rb', line 75

def self.source_code(options: nil)
  new("source_code", nil, options)
end

.unicode_words(options: nil) ⇒ Object 



31
32
33
 
# File 'lib/parade_db/tokenizer.rb', line 31

def self.unicode_words(options: nil)
  new("unicode_words", nil, options)
end

.whitespace(options: nil) ⇒ Object 



27
28
29
 
# File 'lib/parade_db/tokenizer.rb', line 27

def self.whitespace(options: nil)
  new("whitespace", nil, options)
end

Instance Method Details

#render ⇒ Object 



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/parade_db/tokenizer.rb', line 11

def render()
  if options.nil? && positional_args.nil?
    return "pdb.#{name}"
  end

  args = []
  if !positional_args.nil?
    args.concat(positional_args.map { |x| render_positional_arg(x) })
  end
  if !options.nil?
    args.concat(options.map {|k, v| quote_term("#{k}=#{v}")})
  end

  return "pdb.#{name}(#{args.join(",")})"
end