Class: TWPipeline::Sources::HPLT

Inherits:
Object
  • Object
show all
Defined in:
lib/twpipeline/sources/hplt.rb,
sig/twpipeline.rbs

Defined Under Namespace

Classes: Decision

Constant Summary collapse

EXCLUDED_TLDS =

Returns:

  • (::Set[::String]) 
%w[cn hk mo sg my].to_set.freeze
TAIWAN_TLD =

Returns:

  • (::String) 
"tw"
TAIWAN_LANG =

Returns:

  • (::Regexp) 
/\Azh-(?:TW|Hant-TW)\b/i
URL_HEAD =

Returns:

  • (::Integer) 
4096
URL =

Returns:

  • (::Regexp) 
/"u":"([^"]*)"/n
HTML_LANG =

Returns:

  • (::Regexp) 
/"html_lang":\[([^\]]*)\]/n
REGISTERS =

Returns:

  • (::Array[::String]) 
%w[MT LY SP ID NA HI IN OP IP].freeze

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(allowlist: Set.new) ⇒ HPLT

Returns a new instance of HPLT.

Parameters:

  • allowlist: (::Set[::String]) (defaults to: Set.new) 


18
19
20
 
# File 'lib/twpipeline/sources/hplt.rb', line 18

def initialize(allowlist: Set.new)
  @allowlist = allowlist
end

Class Method Details

.allowlist(path) ⇒ ::Set[::String]

Parameters:

  • path (Object)

Returns:

  • (::Set[::String]) 


104
105
106
107
108
 
# File 'lib/twpipeline/sources/hplt.rb', line 104

def allowlist(path)
  return Set.new if path.nil? || !Pathname(path).exist?

  Pathname(path).each_line.map { |line| line.strip.downcase }.reject(&:empty?).to_set
end

.open(path) {|arg0| ... } ⇒ Object

Parameters:

  • path (Object)

Yields:

Yield Parameters:

  • arg0 (::IO)

Yield Returns:

  • (Object)

Returns:

  • (Object) 


99
100
101
102
 
# File 'lib/twpipeline/sources/hplt.rb', line 99

def open(path, &block)
  command = path.to_s.end_with?(".zst") ? ["zstd", "-dc", "-T0", path.to_s] : ["cat", path.to_s]
  IO.popen(command, "rb") { |io| block.call(io) }
end

Instance Method Details

#build(line, decision) ⇒ record

Parameters:

Returns:

  • (record) 


65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/twpipeline/sources/hplt.rb', line 65

def build(line, decision)
  document = Jsonl.parse(line.dup.force_encoding(Encoding::UTF_8).scrub)
  {
    id: document[:id],
    source: "hplt",
    url: document[:u],
    host: decision.host,
    gate: decision.gate.to_s,
    keep: decision.keep,
    crawl: document[:crawl_id],
    html_lang: document[:html_lang],
    lang: Array(document[:lang]).first,
    lang_prob: Array(document[:prob]).first,
    doc_score: score(document),
    register: register(document),
    text: document[:text].to_s,
    ok: true,
    findings: []
  }
end

#decide(line) ⇒ Decision

Parameters:

  • line (::String)

Returns:



33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/twpipeline/sources/hplt.rb', line 33

def decide(line)
  host = host_of(line)
  return Decision.new(keep: false, gate: :excluded, host: host) if host.nil?

  labels = host.split(".")
  tld = labels.last
  return Decision.new(keep: false, gate: :excluded, host: host) if EXCLUDED_TLDS.include?(tld)
  return Decision.new(keep: true, gate: :tld, host: host) if tld == TAIWAN_TLD
  return Decision.new(keep: true, gate: :allowlist, host: host) if @allowlist.include?(host)
  return Decision.new(keep: true, gate: :html_lang, host: host) if taiwan_lang?(line)

  Decision.new(keep: false, gate: :residue, host: host)
end

#each_record(io) ⇒ void #each_record(io) ⇒ ::Enumerator[record, void]

Overloads:

  • #each_record(io) ⇒ void

    This method returns an undefined value.

    Parameters:

    • io (Object)
  • #each_record(io) ⇒ ::Enumerator[record, void]

    Parameters:

    • io (Object)

    Returns:

    • (::Enumerator[record, void])

Yields:

Yield Parameters:

  • arg0 (record)

Yield Returns:

  • (void) 


22
23
24
25
26
27
28
29
30
31
 
# File 'lib/twpipeline/sources/hplt.rb', line 22

def each_record(io)
  return to_enum(:each_record, io) unless block_given?

  io.each_line do |line|
    decision = decide(line)
    next if decision.gate == :excluded

    yield build(line, decision) if decision.keep || decision.gate == :residue
  end
end

#host_of(line) ⇒ ::String?

Parameters:

  • line (::String)

Returns:

  • (::String, nil) 


47
48
49
50
51
52
53
54
 
# File 'lib/twpipeline/sources/hplt.rb', line 47

def host_of(line)
  url = probe(line, URL_HEAD)[URL, 1] || line[URL, 1]
  return nil if url.nil?

  URI.parse(url).host&.downcase&.delete_prefix("www.")
rescue URI::InvalidURIError
  url[%r{\Ahttps?://([^/:?#]+)}i, 1]&.downcase&.delete_prefix("www.")
end

#probe(line, size) ⇒ ::String

Parameters:

  • line (::String)
  • size (::Integer)

Returns:

  • (::String) 


56
 
# File 'lib/twpipeline/sources/hplt.rb', line 56

def probe(line, size) = line.byteslice(0, size).force_encoding(Encoding::BINARY)

#register(document) ⇒ ::String?

Parameters:

  • document (record)

Returns:

  • (::String, nil) 


91
92
93
94
95
96
 
# File 'lib/twpipeline/sources/hplt.rb', line 91

def register(document)
  labels = document[:"web-register"]
  return nil unless labels.is_a?(Hash)

  labels.max_by { |_, value| value.to_f }&.first&.to_s
end

#score(document) ⇒ ::Float?

Parameters:

  • document (record)

Returns:

  • (::Float, nil) 


86
87
88
89
 
# File 'lib/twpipeline/sources/hplt.rb', line 86

def score(document)
  scores = Array(document[:doc_scores]).grep(Numeric)
  scores.empty? ? nil : (scores.sum / scores.length.to_f).round(2)
end

#taiwan_lang?(line) ⇒ Boolean

Parameters:

  • line (::String)

Returns:

  • (Boolean) 


58
59
60
61
62
63
 
# File 'lib/twpipeline/sources/hplt.rb', line 58

def taiwan_lang?(line)
  captured = line[HTML_LANG, 1]
  return false if captured.nil?

  captured.scan(/"([^"]*)"/).flatten.any? { |tag| tag.match?(TAIWAN_LANG) }
end