Module: TWFilter::Checks::Lexicon

Defined in:
lib/twfilter/checks/lexicon.rb,
sig/twfilter.rbs

Constant Summary collapse

NAME =

Returns:

  • (::Symbol) 
:lexicon
ADDRESS =

Taipei streets carry mainland city names by KMT-era decree, so an address suffix suppresses a topic hit.

Returns:

  • (::Regexp) 
/[東西南北中]?(?:[一二三四五六七八九十]段)?(?:路|街|巷|弄|大道|夜市)/

Class Method Summary collapse

Class Method Details

.call(subject) ⇒ ::Array[Finding]

Parameters:

Returns:



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/twfilter/checks/lexicon.rb', line 9

def call(subject)
  findings = []

  hard = hits(subject, hard_terms)
  findings << finding(:mainland, hard.first(DETAIL_SAMPLE).join(" "), :reject) if hard.any?

  severity = subject.policy.soft_lexicon_rejects ? :reject : :mark
  soft = hits(subject, soft_terms)
  findings << finding(:mainland_soft, soft.first(DETAIL_SAMPLE).join(" "), severity) if soft.any?

  cantonese = subject.text.scan(union(:cantonese, cantonese_chars)).uniq
  findings << finding(:cantonese, cantonese.first(DETAIL_SAMPLE).join, :reject) if cantonese.any?

  topics = subject.text.scan(union(:topics, prc_topics)).uniq.select { |term| topic?(subject.text, term) }
  if topics.any?
    severity = subject.policy.prc_topics_reject ? :reject : :mark
    findings << finding(:prc_topic, topics.first(DETAIL_SAMPLE).join(" "), severity)
  end

  findings
end

.cantonese_chars ⇒ ::Array[::String]

Returns:

  • (::Array[::String]) 


41
 
# File 'lib/twfilter/checks/lexicon.rb', line 41

def cantonese_chars = Tables.rows("cantonese.txt")

.exceptions ⇒ ::Hash[::String, ::Array[::String]]

Returns:

  • (::Hash[::String, ::Array[::String]]) 


39
 
# File 'lib/twfilter/checks/lexicon.rb', line 39

def exceptions = Tables.groups("mainland_exceptions.tsv")

.forms(table) ⇒ Object 



37
 
# File 'lib/twfilter/checks/lexicon.rb', line 37

def forms(table) = Tables.columns(table).to_h { |mainland, taiwan, *| [mainland, taiwan] }.freeze

.hard_terms ⇒ ::Hash[::String, ::String]

Returns:

  • (::Hash[::String, ::String]) 


33
 
# File 'lib/twfilter/checks/lexicon.rb', line 33

def hard_terms = @hard_terms ||= forms("mainland_hard.tsv")

.prc_topics ⇒ ::Array[::String]

Returns:

  • (::Array[::String]) 


43
 
# File 'lib/twfilter/checks/lexicon.rb', line 43

def prc_topics = Tables.rows("prc_topics.txt")

.reset! ⇒ Object 



45
 
# File 'lib/twfilter/checks/lexicon.rb', line 45

def reset! = (@hard_terms = @soft_terms = @unions = nil)

.soft_terms ⇒ ::Hash[::String, ::String]

Returns:

  • (::Hash[::String, ::String]) 


35
 
# File 'lib/twfilter/checks/lexicon.rb', line 35

def soft_terms = @soft_terms ||= forms("mainland_soft.tsv")

.taiwan_form(term) ⇒ ::String?

Parameters:

  • (::String)

Returns:

  • (::String, nil) 


31
 
# File 'lib/twfilter/checks/lexicon.rb', line 31

def taiwan_form(term) = hard_terms[term] || soft_terms[term]

.topic?(text, term) ⇒ Boolean

Parameters:

  • (::String)
  • (::String)

Returns:

  • (Boolean) 


50
51
52
53
54
 
# File 'lib/twfilter/checks/lexicon.rb', line 50

def topic?(text, term)
  return false unless text.include?(term)

  text.scan(term).length > text.scan(/#{Regexp.escape(term)}#{ADDRESS}/).length
end

.union(name, terms) ⇒ Object

One union scan per table beats 204 substring searches; the exception mask then runs only for terms that actually matched.



58
 
# File 'lib/twfilter/checks/lexicon.rb', line 58

def union(name, terms) = unions[name] ||= Regexp.union(terms.sort_by { |term| -term.length })