Class: SkillExtractor::Extractor

Inherits:
Object
  • Object
show all
Defined in:
lib/skill_extractor.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(quantized: false) ⇒ Extractor

fp32 (default) matches the reference implementation bit-for-bit



19
20
21
22
23
24
 
# File 'lib/skill_extractor.rb', line 19

def initialize(quantized: false)
  @quantized = quantized
  @matcher = KeywordMatcher.new(JSON.parse(File.read(File.join(DATA_DIR, "skills.json"))))
  @mlp = MLP.new(File.join(DATA_DIR, "mlp.json"))
  @pipe = nil
end

Instance Attribute Details

#matcher ⇒ Object (readonly)

Returns the value of attribute matcher.



26
27
28
 
# File 'lib/skill_extractor.rb', line 26

def matcher
  @matcher
end

#mlp ⇒ Object (readonly)

Returns the value of attribute mlp.



26
27
28
 
# File 'lib/skill_extractor.rb', line 26

def mlp
  @mlp
end

Instance Method Details

#candidates(text) ⇒ Object

Gazetteer matches with their +/-20-word context windows.



29
30
31
32
33
34
35
36
37
38
39
40
 
# File 'lib/skill_extractor.rb', line 29

def candidates(text)
  matches = @matcher.extract(text)
  return [] if matches.empty?

  words = text.split
  matches.map do |skill, start, _end|
    word_idx = text[0...start].split.length
    lo = [0, word_idx - CONTEXT_BEFORE].max
    hi = [words.length, word_idx + CONTEXT_AFTER].min
    { skill: skill, context: words[lo...hi].join(" ") }
  end
end

#classify(inputs) ⇒ Object

P(skill) for each candidate input string.



43
44
45
46
47
48
49
 
# File 'lib/skill_extractor.rb', line 43

def classify(inputs)
  @pipe ||= begin
    require "informers"
    Informers.pipeline("embedding", MODEL, quantized: @quantized)
  end
  @mlp.predict_proba(@pipe.(inputs))
end

#extract(text, threshold: DEFAULT_THRESHOLD) ⇒ Object

Extract confirmed skills from a job posting or resume text.



52
53
54
55
56
57
58
59
60
 
# File 'lib/skill_extractor.rb', line 52

def extract(text, threshold: DEFAULT_THRESHOLD)
  cands = candidates(text)
  return [] if cands.empty?

  probs = classify(cands.map { |c| "#{c[:skill]} : #{c[:context]}" })
  found = {}
  probs.each_with_index { |p, i| found[cands[i][:skill]] = true if p >= threshold }
  found.keys.sort
end