Module: TWFilter::Punctuation

Defined in:
lib/twfilter/punctuation.rb,
sig/twfilter.rbs

Constant Summary collapse

TAIWAN =

Returns:

  • (::Set[::String]) 
"，。、；：？！「」『』（）〔〕【】《》〈〉──……‧—".chars.to_set.freeze
LATIN =

Returns:

  • (::Set[::String]) 
".,;:?!()[]{}\"'-/%&+=*\#@_<>|$^`\\".chars.to_set.freeze
SPACE =

Returns:

  • (::Set[::String]) 
" \t\u{3000}\u{00A0}".chars.to_set.freeze
STRICT =

STRICT is the 教育部 handbook inventory; WIDE adds marks in common Taiwanese use but outside it.

Returns:

  • (::Set[::String]) 
(TAIWAN | LATIN | SPACE | "…⋯．·─".chars.to_set).freeze
WIDE =

Returns:

  • (::Set[::String]) 
(STRICT | "～~℃°※○●–→§±×÷№‼⁉".chars.to_set).freeze
SETS =

Returns:

  • (::Hash[inventory, ::Set[::String]]) 
{strict: STRICT, wide: WIDE}.freeze
ALLOWED = 
WIDE
BOPOMOFO =

Bopomofo and its extended block: the Taiwanese phonetic script, admitted as text.

[0x3100..0x312F, 0x31A0..0x31BF].freeze
TONE_MARKS = 
"\u{02CA}\u{02C7}\u{02CB}\u{02D9}\u{02C9}".chars.to_set.freeze
INVISIBLE = 
/[\u{200B}-\u{200F}\u{2028}\u{2029}\u{FEFF}\u{FE00}-\u{FE0F}\u{00AD}]/
WIDTH = 
{
  "＂" => "\"",
  "＃" => "#",
  "＄" => "$",
  "％" => "%",
  "＆" => "&",
  "＇" => "'",
  "＊" => "*",
  "＋" => "+",
  "－" => "-",
  "．" => ".",
  "／" => "/",
  "＜" => "<",
  "＝" => "=",
  "＞" => ">",
  "＠" => "@",
  "［" => "[",
  "＼" => "\\",
  "］" => "]",
  "＾" => "^",
  "＿" => "_",
  "｀" => "`",
  "｛" => "{",
  "｜" => "|",
  "｝" => "}"
}.freeze
RULES = 
[
  [INVISIBLE, ""],
  [/[#{Regexp.escape(WIDTH.keys.join)}]/, WIDTH],
  # Fullwidth digits and Latin letters fold to ASCII; 0xFEE0 is the fixed offset between the blocks.
  [/[\u{FF10}-\u{FF19}\u{FF21}-\u{FF3A}\u{FF41}-\u{FF5A}]/, -> (m) { (m.ord - 0xFEE0).chr(Encoding::UTF_8) }],
  [/[\u{201C}\u{301D}]/, "「"],
  [/[\u{201D}\u{301E}\u{301F}]/, "」"],
  [/\u{2018}(?=[^\u{2019}]*\p{Han})/, "『"],
  [/(?<=\p{Han})\u{2019}/, "』"],
  [/\u{FE50}/, "，"],
  [/\u{FE55}|\u{FE30}|\u{FE13}/, "："],
  [/[\u{30FB}\u{FF65}]/, "\u{2027}"],
  [/(?<=\p{Han})[\u{00B7}\u{2022}\u{2219}]|[\u{00B7}\u{2022}\u{2219}](?=\p{Han})/, "\u{2027}"],
  [/(?:\u{2026}|\u{22EF}|。。|\.{3,}|・{3,}){1,}/, "\u{2026}\u{2026}"],
  [/[\u{2014}\u{2015}\u{2500}]{1,}|(?<=\p{Han})--(?=\p{Han})/, "\u{2500}\u{2500}"],
  ["\u{301C}", "～"],
  [/(?<=\p{Han}),(?=\s|\p{Han}|\z)/, "，"],
  [/(?<=\p{Han})\.(?=\s|\p{Han}|\z)/, "。"],
  [/(?<=\p{Han})\?(?=\s|\p{Han}|\z)/, "？"],
  [/(?<=\p{Han})!(?=\s|\p{Han}|\z)/, "！"],
  [/(?<=\p{Han});(?=\s|\p{Han}|\z)/, "；"],
  [/(?<=\p{Han}):(?=\s|\p{Han}|\z)/, "："],
  [/(?<=\p{Han})\s+(?=\p{Han})/, ""],
  [/\s+(?=[，。、；：？！」』）〕】》〉])/, ""],
  [/(?<=[，。、；：？！「『（〔【《〈])\s+/, ""],
  [/[ \t\u{00A0}]{2,}/, " "]
].freeze
COLLAPSE = 
/([，。！？；：])\1+/

Class Method Summary collapse

Class Method Details

.acceptable?(char, set: WIDE) ⇒ Boolean

Parameters:

  • (::String)
  • set: (::Set[::String]) (defaults to: WIDE)

Returns:

  • (Boolean) 


93
94
95
96
97
98
99
 
# File 'lib/twfilter/punctuation.rb', line 93

def acceptable?(char, set: WIDE)
  return true if Han.char?(char) || set.include?(char) || TONE_MARKS.include?(char)
  return true if char.match?(/[A-Za-z0-9]/)

  code = char.ord
  BOPOMOFO.any? { |range| range.cover?(code) }
end

.normalize(text, collapse_repeats: true) ⇒ ::String

Parameters:

  • (Object)
  • collapse_repeats: (Boolean) (defaults to: true)

Returns:

  • (::String) 


78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/twfilter/punctuation.rb', line 78

def normalize(text, collapse_repeats: true)
  value = RULES.reduce(text.to_s.unicode_normalize(:nfc)) { |memo, (pattern, replacement)|
    case replacement
    in Proc
      memo.gsub(pattern) { replacement.call(Regexp.last_match(0)) }
    in Hash
      memo.gsub(pattern, replacement)
    in String
      memo.gsub(pattern, replacement)
    end
  }
  value = value.gsub(COLLAPSE, "\\1") if collapse_repeats
  value.strip
end

.offenders(text, punctuation: :wide) ⇒ ::Array[::String]

Parameters:

  • (::String)
  • punctuation: (inventory) (defaults to: :wide)

Returns:

  • (::Array[::String]) 


101
102
103
104
 
# File 'lib/twfilter/punctuation.rb', line 101

def offenders(text, punctuation: :wide)
  set = SETS.fetch(punctuation)
  text.each_char.reject { |char| acceptable?(char, set: set) }.uniq
end