Module: FetchUtil::Regulatory::Human

Included in:
FetchUtil::Regulatory
Defined in:
lib/fetch_util/regulatory/human.rb

Constant Summary collapse

HUMAN_PATTERNS = 
[
  {
    "verb" => "disallow",
    "noun" => "text-and-data-mining",
    "regex" => /
      \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
      \b(?:text\s+and\s+data\s+mining|text-and-data-mining|tdm)\b
    /ix
  },
  {
    "verb" => "disallow",
    "noun" => "ai-training",
    "regex" => /
      \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
      \b(?:ai\s+training|training\s+(?:of|for)\s+(?:ai|models?)|machine\s+learning|large\s+language\s+model|large\s+language\s+models|llm|generative\s+ai)\b
    /ix
  },
  {
    "verb" => "disallow",
    "noun" => "index",
    "regex" => /
      \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
      \b(?:index(?:ing)?|search\s+engine(?:s)?)\b
    /ix
  },
  {
    "verb" => "disallow",
    "noun" => "fetch",
    "regex" => /
      \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
      \b(?:crawl(?:ing)?|scrap(?:e|ing)|fetch(?:ing)?|bot(?:s)?)\b
    /ix
  },
  {
    "verb" => "allow",
    "noun" => "text-and-data-mining",
    "regex" => /\b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}\b(?:text\s+and\s+data\s+mining|text-and-data-mining|tdm)\b/i
  },
  {
    "verb" => "allow",
    "noun" => "ai-training",
    "regex" => /
      \b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}
      \b(?:ai\s+training|training\s+(?:of|for)\s+(?:ai|models?)|machine\s+learning|large\s+language\s+model|large\s+language\s+models|llm|generative\s+ai)\b
    /ix
  },
  {
    "verb" => "allow",
    "noun" => "index",
    "regex" => /\b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}\b(?:index(?:ing)?|search\s+engine(?:s)?)\b/i
  },
  {
    "verb" => "allow",
    "noun" => "fetch",
    "regex" => /\b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}\b(?:crawl(?:ing)?|scrap(?:e|ing)|fetch(?:ing)?|bot(?:s)?)\b/i
  }
].freeze

Instance Method Summary collapse

Instance Method Details

#extract_human_signals(body, path:) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/fetch_util/regulatory/human.rb', line 64

def extract_human_signals(body, path:)
  chunks = human_text_chunks(body)
  seen = {}

  HUMAN_PATTERNS.filter_map do |entry|
    evidence = chunks.find { |chunk| chunk.match?(entry["regex"]) }
    next unless evidence
    next if entry["verb"] == "allow" && negative_human_chunk?(evidence)

    key = [entry["verb"], entry["noun"], evidence]
    next if seen[key]

    seen[key] = true
    build_signal(
      entry["verb"],
      entry["noun"],
      path: path,
      conditions: { "evidence" => evidence }
    )
  end
end