Module: Tina4::ProjectIndex

Defined in:
lib/tina4/project_index.rb

Constant Summary collapse

INDEX_DIRNAME = 
".tina4"
INDEX_FILENAME = 
"project_index.json"
SKIP_DIRS = 
%w[
  .git .hg .svn node_modules __pycache__ .venv venv .mypy_cache
  .ruff_cache .pytest_cache dist build .tina4 logs .idea .vscode
  vendor coverage tmp .bundle
].freeze
INDEX_EXT = 
%w[
  .rb .erb .twig .html .sql .scss .css .js .ts .mjs .md .json .yml
  .yaml .toml .env .rake
].freeze
MAX_FILE_BYTES = 
256 * 1024
ROUTE_METHODS = 
%w[get post put patch delete any any_method secure_get secure_post].freeze
TWIG_EXTENDS = 
/\{%\s*extends\s+['"]([^'"]+)['"]\s*%\}/.freeze
TWIG_BLOCK = 
/\{%\s*block\s+([A-Za-z_][\w-]*)/.freeze
TWIG_INCLUDE = 
/\{%\s*include\s+['"]([^'"]+)['"]/.freeze
SQL_CREATE = 
/create\s+(?:unique\s+)?(table|index|view|trigger|sequence|procedure|function)\s+(?:if\s+not\s+exists\s+)?([A-Za-z_][\w.]*)/i.freeze
SQL_ALTER = 
/alter\s+(table|index|view)\s+([A-Za-z_][\w.]*)/i.freeze
JS_EXPORT = 
/^\s*export\s+(?:default\s+)?(?:async\s+)?(?:function|class|const|let|var|interface|type|enum)\s+([A-Za-z_$][\w$]*)/.freeze
JS_IMPORT = 
/^\s*import\s+[^'"]+?['"]([^'"]+)['"]/.freeze
MD_H1 = 
/^#\s+(.+)$/.freeze
MD_H2 = 
/^##\s+(.+)$/.freeze
EXTRACTORS = 
{
  ".rb"   => :extract_ruby,
  ".rake" => :extract_ruby,
  ".erb"  => :extract_erb,
  ".twig" => :extract_twig,
  ".html" => :extract_twig,
  ".sql"  => :extract_sql,
  ".js"   => :extract_js_ts,
  ".mjs"  => :extract_js_ts,
  ".ts"   => :extract_js_ts,
  ".md"   => :extract_md
}.freeze
LANGUAGES = 
{
  ".rb" => "ruby", ".rake" => "ruby",
  ".erb" => "erb", ".twig" => "twig", ".html" => "html",
  ".sql" => "sql", ".scss" => "scss", ".css" => "css",
  ".js" => "javascript", ".mjs" => "javascript", ".ts" => "typescript",
  ".md" => "markdown", ".json" => "json", ".yml" => "yaml",
  ".yaml" => "yaml", ".toml" => "toml", ".env" => "env"
}.freeze

Class Method Summary collapse

Class Method Details

.extract(path) ⇒ Object

── Index core ───────────────────────────────────────────



163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/tina4/project_index.rb', line 163

def extract(path)
  stat = File.stat(path)
  rel  = path.sub("#{project_root}/", "")
  entry = {
    "path"     => rel,
    "size"     => stat.size,
    "mtime"    => stat.mtime.to_i,
    "language" => language_for(path)
  }
  return entry.merge("skipped" => "too large (#{stat.size} bytes)") if stat.size > MAX_FILE_BYTES

  text = begin
    File.read(path, encoding: "utf-8", invalid: :replace, undef: :replace)
  rescue StandardError
    return entry
  end
  entry["sha256"] = Digest::SHA256.hexdigest(text)[0, 16]
  ext = File.extname(path)
  extractor = EXTRACTORS[ext]
  begin
    data = extractor ? send(extractor, text) : extract_generic(text)
    entry.merge!(data) if data.is_a?(Hash)
  rescue StandardError => e
    entry["extraction_error"] = e.message[0, 200]
  end
  entry["summary"] = summarise(entry)
  entry
rescue Errno::ENOENT, Errno::EACCES
  {}
end

.extract_erb(text) ⇒ Object 



90
91
92
93
94
 
# File 'lib/tina4/project_index.rb', line 90

def extract_erb(text)
  # ERB acts a lot like Twig here — extract partial renders.
  renders = text.scan(/render\s+['"]([^'"]+)['"]/).flatten.uniq
  { "renders" => renders }
end

.extract_generic(text) ⇒ Object 



126
127
128
129
130
131
132
133
 
# File 'lib/tina4/project_index.rb', line 126

def extract_generic(text)
  text.each_line do |line|
    s = line.strip
    next if s.empty? || s.start_with?("<!--")
    return { "first_line" => s[0, 200] }
  end
  {}
end

.extract_js_ts(text) ⇒ Object 



109
110
111
112
113
114
 
# File 'lib/tina4/project_index.rb', line 109

def extract_js_ts(text)
  {
    "exports" => text.scan(JS_EXPORT).flatten.uniq.sort,
    "imports" => text.scan(JS_IMPORT).flatten.uniq.sort
  }
end

.extract_md(text) ⇒ Object 



119
120
121
122
123
124
 
# File 'lib/tina4/project_index.rb', line 119

def extract_md(text)
  {
    "title"    => (text[MD_H1, 1] || "").strip,
    "sections" => text.scan(MD_H2).flatten.first(30)
  }
end

.extract_ruby(text) ⇒ Object

── Extractors ────────────────────────────────────────────



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/tina4/project_index.rb', line 48

def extract_ruby(text)
  out = { "symbols" => [], "imports" => [], "routes" => [], "docstring" => "" }
  # Pick up first non-blank comment block as pseudo-docstring.
  first_comment = nil
  text.each_line do |ln|
    s = ln.strip
    next if s.empty? || s.start_with?("#!") || s == "# frozen_string_literal: true"
    if s.start_with?("#")
      first_comment = s.sub(/\A#\s*/, "")[0, 200]
      break
    else
      break
    end
  end
  out["docstring"] = first_comment if first_comment

  text.scan(/^\s*(?:class|module)\s+([A-Z][\w:]*)/) { |m| out["symbols"] << m[0] }
  text.scan(/^\s*def\s+(self\.)?([A-Za-z_][\w!?=]*)/) { |m| out["symbols"] << m[1] }
  text.scan(/^\s*require(?:_relative)?\s+['"]([^'"]+)['"]/) { |m| out["imports"] << m[0] }
  # Tina4.get "/path"  OR  Tina4::Router.get("/path")  OR  get "/path" do
  route_re = /(?:Tina4(?:::Router)?\.|^\s*)(get|post|put|patch|delete|any|any_method|secure_get|secure_post)\s*\(?\s*['"]([^'"]+)['"]/
  text.scan(route_re) do |meth, path|
    next unless ROUTE_METHODS.include?(meth)
    out["routes"] << { "method" => meth.upcase, "path" => path, "handler" => "" }
  end
  out["symbols"].uniq!
  out["imports"].uniq!
  out
end

.extract_sql(text) ⇒ Object 



99
100
101
102
103
104
 
# File 'lib/tina4/project_index.rb', line 99

def extract_sql(text)
  out = { "creates" => [], "alters" => [] }
  text.scan(SQL_CREATE) { |kind, name| out["creates"] << "#{kind.upcase} #{name}" }
  text.scan(SQL_ALTER)  { |kind, name| out["alters"]  << "#{kind.upcase} #{name}" }
  out
end

.extract_twig(text) ⇒ Object 



82
83
84
85
86
87
88
 
# File 'lib/tina4/project_index.rb', line 82

def extract_twig(text)
  {
    "extends"  => text.scan(TWIG_EXTENDS).flatten,
    "blocks"   => text.scan(TWIG_BLOCK).flatten.uniq.sort,
    "includes" => text.scan(TWIG_INCLUDE).flatten.uniq.sort
  }
end

.file_entry(rel_path) ⇒ Object 



330
331
332
333
334
335
 
# File 'lib/tina4/project_index.rb', line 330

def file_entry(rel_path)
  refresh
  data = load_raw
  entry = data["files"][rel_path]
  entry || { "error" => "Not in index: #{rel_path}" }
end

.index_path ⇒ Object 



40
41
42
43
44
 
# File 'lib/tina4/project_index.rb', line 40

def index_path
  dir = File.join(project_root, INDEX_DIRNAME)
  FileUtils.mkdir_p(dir)
  File.join(dir, INDEX_FILENAME)
end

.language_for(path) ⇒ Object 



157
158
159
 
# File 'lib/tina4/project_index.rb', line 157

def language_for(path)
  LANGUAGES[File.extname(path)] || "text"
end

.load_raw ⇒ Object 



240
241
242
243
244
245
246
 
# File 'lib/tina4/project_index.rb', line 240

def load_raw
  p = index_path
  return { "version" => 1, "files" => {}, "generated_at" => 0 } unless File.exist?(p)
  JSON.parse(File.read(p, encoding: "utf-8"))
rescue StandardError
  { "version" => 1, "files" => {}, "generated_at" => 0 }
end

.overview ⇒ Object 



337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
 
# File 'lib/tina4/project_index.rb', line 337

def overview
  refresh
  data = load_raw
  files = data["files"]
  langs = Hash.new(0)
  route_count = 0
  model_count = 0
  files.each_value do |e|
    langs[e["language"] || "other"] += 1
    route_count += (e["routes"] || []).size
    path = e["path"].to_s
    if (path.start_with?("src/orm/") || path.start_with?("orm/") || path.start_with?("app/models/")) && (e["symbols"] && !e["symbols"].empty?)
      model_count += 1
    end
  end
  recent = files.values.map do |e|
    { "path" => e["path"], "summary" => e["summary"] || "", "mtime" => e["mtime"] || 0 }
  end.sort_by { |e| -(e["mtime"] || 0) }.first(10)
  {
    "total_files"        => files.size,
    "by_language"        => langs,
    "routes_declared"    => route_count,
    "orm_models"         => model_count,
    "recently_changed"   => recent,
    "index_generated_at" => data["generated_at"] || 0
  }
end

.project_root ⇒ Object 



36
37
38
 
# File 'lib/tina4/project_index.rb', line 36

def project_root
  File.expand_path(Dir.pwd)
end

.refresh ⇒ Object 



253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
 
# File 'lib/tina4/project_index.rb', line 253

def refresh
  data = load_raw
  files = data["files"] || {}
  added = 0
  updated = 0
  seen = {}
  root = project_root

  walk_project.each do |p|
    rel = p.sub("#{root}/", "")
    seen[rel] = true
    begin
      mtime = File.mtime(p).to_i
    rescue Errno::ENOENT
      next
    end
    existing = files[rel]
    if existing && existing["mtime"] == mtime
      next
    end
    files[rel] = extract(p)
    if existing
      updated += 1
    else
      added += 1
    end
  end

  removed_paths = files.keys - seen.keys
  removed_paths.each { |k| files.delete(k) }
  data["files"] = files
  save_raw(data)
  {
    "added"   => added,
    "updated" => updated,
    "removed" => removed_paths.size,
    "total"   => files.size,
    "path"    => index_path.sub("#{root}/", "")
  }
end

.save_raw(data) ⇒ Object 



248
249
250
251
 
# File 'lib/tina4/project_index.rb', line 248

def save_raw(data)
  data["generated_at"] = Time.now.to_i
  File.write(index_path, JSON.pretty_generate(data), encoding: "utf-8")
end

.search(query, limit = 20) ⇒ Object 



294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/tina4/project_index.rb', line 294

def search(query, limit = 20)
  refresh
  data = load_raw
  q = query.to_s.downcase.strip
  return [] if q.empty?
  hits = []
  data["files"].each do |rel, entry|
    score = 0
    score += 10 if rel.downcase.include?(q)
    (entry["symbols"] || []).each do |s|
      sl = s.downcase
      if sl == q
        score += 8
      elsif sl.include?(q)
        score += 4
      end
    end
    (entry["routes"] || []).each do |r|
      combined = "#{r["path"]} #{r["handler"]}".downcase
      score += 5 if combined.include?(q)
    end
    score += 3 if (entry["summary"] || "").downcase.include?(q)
    (entry["imports"] || []).each { |imp| score += 1 if imp.downcase.include?(q) }
    if score.positive?
      hits << [score, {
        "path"     => rel,
        "summary"  => entry["summary"] || "",
        "score"    => score,
        "language" => entry["language"] || ""
      }]
    end
  end
  hits.sort_by! { |h| -h[0] }
  hits.first([1, limit].max).map { |_, info| info }
end

.summarise(entry) ⇒ Object 



194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
 
# File 'lib/tina4/project_index.rb', line 194

def summarise(entry)
  return entry["skipped"] if entry["skipped"]
  return entry["docstring"] if entry["docstring"] && !entry["docstring"].to_s.empty?
  return entry["title"]     if entry["title"] && !entry["title"].to_s.empty?
  if entry["routes"] && !entry["routes"].empty?
    r = entry["routes"][0]
    extra = entry["routes"].size > 1 ? " (+#{entry["routes"].size - 1} more)" : ""
    return "#{r["method"]} #{r["path"]}#{extra}"
  end
  return "defines " + entry["symbols"].first(4).join(", ") if entry["symbols"] && !entry["symbols"].empty?
  return "exports " + entry["exports"].first(4).join(", ") if entry["exports"] && !entry["exports"].empty?
  return "schema: " + entry["creates"].first(3).join(", ") if entry["creates"] && !entry["creates"].empty?
  return "template, extends #{entry["extends"][0]}" if entry["extends"] && !entry["extends"].empty?
  return entry["first_line"] if entry["first_line"]
  ""
end

.walk_project ⇒ Object 



211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
 
# File 'lib/tina4/project_index.rb', line 211

def walk_project
  root = project_root
  found = []
  prefix_len = root.length + 1
  walker = lambda do |dir|
    Dir.each_child(dir) do |name|
      next if name == "." || name == ".."
      # Skip hidden dirs (allow .env as a file below)
      full = File.join(dir, name)
      if File.directory?(full)
        next if SKIP_DIRS.include?(name)
        next if name.start_with?(".")
        walker.call(full)
      elsif File.file?(full)
        if name.start_with?(".")
          next unless name == ".env"
        end
        ext = File.extname(name)
        next unless INDEX_EXT.include?(ext) || name == ".env"
        found << full
      end
    end
  rescue Errno::EACCES, Errno::ENOENT
    # skip
  end
  walker.call(root)
  found
end