Module: Jekyll::VitePressTheme::LLMS

Defined in:
lib/jekyll/vitepress_theme/llms.rb

Overview

Generates the proposed llms.txt index and its full-text companion from the same canonical, indexable content used by the theme's SEO layer. rubocop:disable Metrics/AbcSize, Metrics/ModuleLength

Defined Under Namespace

Classes: GeneratedPage

Class Method Summary collapse

Class Method Details

.add_group(groups, used, title, items) ⇒ Object 



197
198
199
200
201
202
203
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 197

def add_group(groups, used, title, items)
  items = items.reject { |item| used[item] }
  return if items.empty?

  items.each { |item| used[item] = true }
  groups << { title: title, items: items }
end

.apply(site) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 27

def apply(site)
  config = config_for(site)
  return unless enabled?(config)

  groups = content_groups(site)
  return if groups.empty?

  unless SEO.custom_output?(site, 'llms.txt')
    site.pages << GeneratedPage.new(site, 'llms.txt', render_index(site, config, groups))
  end

  return unless full_text?(config) && !SEO.custom_output?(site, 'llms-full.txt')

  site.pages << GeneratedPage.new(site, 'llms-full.txt', render_full(site, config, groups))
rescue StandardError => e
  Jekyll.logger.warn('jekyll-vitepress-theme', "LLM discovery generation failed: #{e.message}")
end

.canonical_url(item) ⇒ Object 



170
171
172
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 170

def canonical_url(item)
  item.data.dig('_seo', 'canonical_url') || site_url(item.site, item.url)
end

.collection_label(item) ⇒ Object 



212
213
214
215
216
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 212

def collection_label(item)
  return unless item.respond_to?(:collection) && item.collection

  item.collection.label
end

.collection_title(label) ⇒ Object 



218
219
220
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 218

def collection_title(label)
  label.to_s.tr('_-', ' ').split.map(&:capitalize).join(' ')
end

.config_for(site) ⇒ Object 



45
46
47
48
49
50
51
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 45

def config_for(site)
  theme_config = site.config['jekyll_vitepress']
  return {} unless theme_config.is_a?(Hash)

  value = theme_config.fetch('llms', {})
  value == false ? false : hash_value(value)
end

.content_groups(site) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 61

def content_groups(site)
  eligible = SEO.html_items(site).select { |item| eligible?(item) }
  used = {}.compare_by_identity
  groups = []

  home = eligible.find { |item| item.url == '/' }
  add_group(groups, used, 'Overview', [home].compact)

  sidebar_groups(site).each do |group|
    docs = Array(group['docs']).select { |item| eligible.include?(item) }
    add_group(groups, used, group['title'] || collection_title(group['collection']), docs)
  end

  remaining = eligible.reject { |item| used[item] }
  remaining.group_by { |item| collection_label(item) }.each do |label, items|
    title = label ? collection_title(label) : 'Pages'
    add_group(groups, used, title, sorted_items(items))
  end

  groups
end

.eligible?(item) ⇒ Boolean

Returns:

  • (Boolean) 


83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 83

def eligible?(item)
  return false if item.data['llms'] == false || item.data['redirect_to']
  return false if ['/404.html', '/404/'].include?(item.url)

  metadata = item.data['_seo']
  robots = metadata&.dig('robots') || item.data['robots']
  tokens = robots.to_s.downcase.split(/[\s,]+/)
  return false if item.data['noindex'] == true || tokens.intersect?(%w[noindex none])

  canonical = metadata&.dig('canonical_url')
  self_url = SEO.absolute_url(item.site, item.url)
  canonical.to_s.empty? || self_url.to_s.empty? || canonical == self_url
end

.enabled?(config) ⇒ Boolean

Returns:

  • (Boolean) 


53
54
55
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 53

def enabled?(config)
  config != false && config.fetch('enabled', true) != false
end

.full_text?(config) ⇒ Boolean

Returns:

  • (Boolean) 


57
58
59
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 57

def full_text?(config)
  config.fetch('full', config.fetch('include_full_text', true)) != false
end

.hash_value(value) ⇒ Object 



230
231
232
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 230

def hash_value(value)
  value.is_a?(Hash) ? value : {}
end

.header_lines(site, config) ⇒ Object 



147
148
149
150
151
152
153
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 147

def header_lines(site, config)
  title = config['title'] || SEO.site_title(site) || 'Documentation'
  description = config['description'] || site.config['description']
  lines = ["# #{plain_heading(title)}", '']
  lines.push("> #{SEO.clean_text(description)}", '') if SEO.clean_text(description)
  lines
end

.item_description(item) ⇒ Object 



188
189
190
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 188

def item_description(item)
  SEO.clean_text(item.data.dig('_seo', 'description') || item.data['description'])
end

.item_title(item) ⇒ Object 



184
185
186
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 184

def item_title(item)
  item.data.dig('_seo', 'page_title') || SEO.clean_text(item.data['title']) || item.url
end

.link_label(value) ⇒ Object 



226
227
228
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 226

def link_label(value)
  plain_heading(value).gsub(/([\[\]])/, '\\\1')
end

.plain_heading(value) ⇒ Object 



222
223
224
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 222

def plain_heading(value)
  SEO.clean_text(value).to_s.gsub(/[\r\n#]+/, ' ').strip
end

.raw_markdown(item, site) ⇒ Object 



155
156
157
158
159
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 155

def raw_markdown(item, site)
  payload = site.site_payload.merge('page' => item.to_liquid)
  raw = CopyPage.resolved_markdown(item, payload)
  CopyPage.with_title(raw, item_title(item)).to_s.strip
end

.relative_url(site, path) ⇒ Object 



178
179
180
181
182
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 178

def relative_url(site, path)
  baseurl = site.config['baseurl'].to_s.sub(%r{/+\z}, '')
  path = "/#{path}" unless path.start_with?('/')
  "#{baseurl}#{path}"
end

.render_full(site, config, groups) ⇒ Object 



125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 125

def render_full(site, config, groups)
  title = config['title'] || SEO.site_title(site) || 'Documentation'
  description = config['description'] || site.config['description']
  lines = ["# #{plain_heading(title)} — Full Documentation", '']
  lines.push("> #{SEO.clean_text(description)}", '') if SEO.clean_text(description)

  groups.flat_map { |group| group[:items] }.each do |item|
    lines << '---'
    lines << ''
    lines << "# #{plain_heading(item_title(item))}"
    lines << ''
    lines << "Canonical URL: #{canonical_url(item)}"
    lines << ''
    content = strip_leading_title(raw_markdown(item, site))
    content = item_description(item).to_s if content.empty?
    lines << content
    lines << ''
  end

  "#{lines.join("\n").rstrip}\n"
end

.render_index(site, config, groups) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 97

def render_index(site, config, groups)
  lines = header_lines(site, config)
  details = config['details'].to_s.strip
  lines.push(details, '') unless details.empty?

  if full_text?(config)
    lines << '## Full Documentation'
    lines << ''
    lines << "- [Complete documentation](#{site_url(site, '/llms-full.txt')}): All canonical pages in one Markdown document."
    lines << ''
  end

  groups.each do |group|
    lines << "## #{plain_heading(group[:title])}"
    lines << ''
    group[:items].each do |item|
      title = link_label(item_title(item))
      entry = "- [#{title}](#{canonical_url(item)})"
      description = item_description(item)
      entry += ": #{description}" if description
      lines << entry
    end
    lines << ''
  end

  "#{lines.join("\n").rstrip}\n"
end

.sidebar_groups(site) ⇒ Object 



192
193
194
195
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 192

def sidebar_groups(site)
  data = site.data['jekyll_vitepress_sidebar']
  data.is_a?(Hash) ? Array(data['groups']) : []
end

.site_url(site, path) ⇒ Object 



174
175
176
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 174

def site_url(site, path)
  SEO.absolute_url(site, path) || relative_url(site, path)
end

.sorted_items(items) ⇒ Object 



205
206
207
208
209
210
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 205

def sorted_items(items)
  items.sort_by do |item|
    order = item.data['nav_order']
    [order.is_a?(Numeric) ? order : Float::INFINITY, item.url.to_s]
  end
end

.strip_leading_title(markdown) ⇒ Object 



161
162
163
164
165
166
167
168
 
# File 'lib/jekyll/vitepress_theme/llms.rb', line 161

def strip_leading_title(markdown)
  markdown.to_s
          .sub(/\A\s*#\s+[^\n]+\n+/, '')
          .sub(%r{\A\s*<h1(?:\s[^>]*)?>.*?</h1>\s*}mi, '')
          .sub(/\A\s*[^\n]+\n=+\s*\n+/, '')
          .gsub(/\n{3,}/, "\n\n")
          .strip
end