Class: Mxrb::IO::MprFile

Inherits:
Object
  • Object
show all
Defined in:
lib/mxrb/io/mpr_file.rb

Overview

Low-level SQLite wrapper for .mpr files.

Schema reality (from reverse engineering):

Unit(UnitID BLOB, ContainerID BLOB, ContainmentName TEXT,
   TreeConflict LONG, ContentsHash TEXT, ContentsConflict TEXT, Contents BLOB)
_MetaData(_ProductVersion TEXT, _BuildVersion TEXT, _SchemaHash TEXT)
(older MPRs use MendixVersion instead of _ProductVersion)

UnitID and ContainerID are 16-byte MS-GUID blobs, not integers. Contents is a BSON blob (v1) or NULL (v2, where .mxunit files hold the data). The $Type of each unit is embedded in its BSON Contents, not in a separate table.

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(path, readonly: false) ⇒ MprFile

Returns a new instance of MprFile.



31
32
33
34
35
36
37
38
 
# File 'lib/mxrb/io/mpr_file.rb', line 31

def initialize(path, readonly: false)
  @path     = File.expand_path(path)
  @readonly = readonly
  @db       = open_db
  @write_stats = { inserted: 0, updated: 0, skipped: 0, deleted: 0 }
  validate!
  @format_version = detect_format
end

Instance Attribute Details

#format_version ⇒ Object (readonly)

Returns the value of attribute format_version.



23
24
25
 
# File 'lib/mxrb/io/mpr_file.rb', line 23

def format_version
  @format_version
end

#path ⇒ Object (readonly)

Returns the value of attribute path.



23
24
25
 
# File 'lib/mxrb/io/mpr_file.rb', line 23

def path
  @path
end

Class Method Details

.open(path, readonly: false) ⇒ Object 



40
41
42
 
# File 'lib/mxrb/io/mpr_file.rb', line 40

def self.open(path, readonly: false)
  new(path, readonly: readonly)
end

Instance Method Details

#all_units ⇒ Object

All units (for exploration / reverse engineering).



120
121
122
123
124
 
# File 'lib/mxrb/io/mpr_file.rb', line 120

def all_units
  @db.execute(
    "SELECT #{unit_select_columns} FROM Unit"
  ).map { raw_to_hash(_1) }
end

#architecture_definition ⇒ Object

mxrb-only architecture metadata for concepts without a native Mendix unit (ports/repositories) or bindings awaiting a concrete widget tree.



463
464
465
466
467
 
# File 'lib/mxrb/io/mpr_file.rb', line 463

def architecture_definition
  return nil unless tables.include?("_MxrbArchitecture")
  json = @db.get_first_value("SELECT Definition FROM _MxrbArchitecture WHERE ID = 1")
  json && JSON.parse(json, symbolize_names: true)
end

#backup!(dest_path) ⇒ Object

Creates a consistent point-in-time backup using SQLite's VACUUM INTO. Falls back to a WAL checkpoint + file copy on older SQLite versions.



253
254
255
256
257
258
259
260
261
 
# File 'lib/mxrb/io/mpr_file.rb', line 253

def backup!(dest_path)
  raise ReadOnlyError, "Opened in read-only mode" if @readonly

  FileUtils.rm_f(dest_path)
  @db.execute("VACUUM INTO ?", [dest_path])
rescue SQLite3::Exception
  @db.execute("PRAGMA wal_checkpoint(FULL)") rescue nil
  FileUtils.cp(@path, dest_path)
end

#children_of(parent_uuid) ⇒ Object

Units directly contained by a given parent UUID.



100
101
102
103
104
105
106
107
 
# File 'lib/mxrb/io/mpr_file.rb', line 100

def children_of(parent_uuid)
  blob = BsonCodec.uuid_to_blob(parent_uuid)
  @db.execute(
    "SELECT #{unit_select_columns} FROM Unit " \
    "WHERE ContainerID = ? AND UnitID != ContainerID",
    [blob]
  ).map { raw_to_hash(_1) }
end

#clear_index_cache! ⇒ Object

Clears cached semantic data while preserving the cache table.

Raises:



355
356
357
358
359
360
361
362
 
# File 'lib/mxrb/io/mpr_file.rb', line 355

def clear_index_cache!
  raise ReadOnlyError, "Opened in read-only mode" if @readonly
  return 0 unless tables.include?("_MxrbIndexCache")

  count = @db.get_first_value("SELECT COUNT(*) FROM _MxrbIndexCache").to_i
  @db.execute("DELETE FROM _MxrbIndexCache")
  count
end

#close ⇒ Object 



484
485
486
 
# File 'lib/mxrb/io/mpr_file.rb', line 484

def close
  @db.close
end

#content_bytes(raw_unit) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
 
# File 'lib/mxrb/io/mpr_file.rb', line 139

def content_bytes(raw_unit)
  blob = raw_unit["Contents"]
  if (blob.nil? || blob.empty?) && @format_version == :v2
    unit_path = MxunitCodec.path_for(contents_dir, raw_unit.fetch("UnitID"))
    return nil unless File.file?(unit_path)

    return File.binread(unit_path)
  end
  blob
end

#content_files ⇒ Object 



156
157
158
159
160
 
# File 'lib/mxrb/io/mpr_file.rb', line 156

def content_files
  return [] unless @format_version == :v2 && File.directory?(contents_dir)

  Dir.glob(File.join(contents_dir, "**", "*.mxunit")).sort
end

#content_path(raw_unit) ⇒ Object 



150
151
152
153
154
 
# File 'lib/mxrb/io/mpr_file.rb', line 150

def content_path(raw_unit)
  return nil unless @format_version == :v2

  MxunitCodec.path_for(contents_dir, raw_unit.fetch("UnitID"))
end

#delete_unit(uuid) ⇒ Object

Raises:



226
227
228
229
230
231
232
 
# File 'lib/mxrb/io/mpr_file.rb', line 226

def delete_unit(uuid)
  raise ReadOnlyError, "Opened in read-only mode" if @readonly
  @db.execute("DELETE FROM Unit WHERE UnitID = ?", [BsonCodec.uuid_to_blob(uuid)])
  removed = FileUtils.rm_f(MxunitCodec.path_for(contents_dir, uuid)) if @format_version == :v2
  write_stats[:deleted] += 1
  removed
end

#ensure_vec_meta_table!(table) ⇒ Object 



392
393
394
395
396
397
398
399
400
401
402
403
 
# File 'lib/mxrb/io/mpr_file.rb', line 392

def ensure_vec_meta_table!(table)
  ensure_vector_write!
  identifier = vector_identifier(table)
  @db.execute(<<~SQL)
    CREATE TABLE IF NOT EXISTS #{identifier} (
      ID INTEGER PRIMARY KEY CHECK (ID = 1),
      Backend TEXT NOT NULL,
      Dimension INTEGER NOT NULL,
      Fingerprint TEXT NOT NULL
    )
  SQL
end

#ensure_vec_table!(table, dimension) ⇒ Object

Raises:

  • (ArgumentError) 


380
381
382
383
384
385
386
387
388
389
390
 
# File 'lib/mxrb/io/mpr_file.rb', line 380

def ensure_vec_table!(table, dimension)
  ensure_vector_write!
  identifier = vector_identifier(table)
  size = Integer(dimension)
  raise ArgumentError, "vector dimension must be positive" unless size.positive?

  @db.execute(<<~SQL)
    CREATE VIRTUAL TABLE IF NOT EXISTS #{identifier}
    USING vec0(artifact_id TEXT PRIMARY KEY, embedding FLOAT[#{size}])
  SQL
end

#index_cache_info(current_fingerprint: nil) ⇒ Object

Returns cache size and fingerprints without parsing the cached payload.



325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
 
# File 'lib/mxrb/io/mpr_file.rb', line 325

def index_cache_info(current_fingerprint: nil)
  unless tables.include?("_MxrbIndexCache")
    return {
      present: false, entries: 0, bytes: 0,
      fingerprints: [].freeze, current_fingerprint:,
      hit: false
    }.freeze
  end

  rows = @db.execute(
    "SELECT Fingerprint, LENGTH(IndexData) FROM _MxrbIndexCache ORDER BY Fingerprint"
  )
  fingerprints = rows.map { _1[0].to_s }.freeze
  {
    present: !rows.empty?,
    entries: rows.size,
    bytes: rows.sum { _1[1].to_i },
    fingerprints:,
    current_fingerprint:,
    hit: current_fingerprint && fingerprints.include?(current_fingerprint)
  }.freeze
rescue SQLite3::Exception
  {
    present: false, entries: 0, bytes: 0,
    fingerprints: [].freeze, current_fingerprint:,
    hit: false
  }.freeze
end

#insert_unit(container_uuid:, containment_name:, contents_doc:) ⇒ Object

Insert a new unit. Returns the assigned UUID.

Raises:



165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/mxrb/io/mpr_file.rb', line 165

def insert_unit(container_uuid:, containment_name:, contents_doc:)
  raise ReadOnlyError, "Opened in read-only mode" if @readonly

  uuid = BsonCodec.extract_id(contents_doc["$ID"] || contents_doc["\$ID"]) || SecureRandom.uuid
  unless contents_doc.key?("$ID") || contents_doc.key?("\$ID")
    contents_doc = { "$ID" => uuid }.merge(contents_doc)
  end
  unit_blob    = BsonCodec.uuid_to_blob(uuid)
  parent_blob  = BsonCodec.uuid_to_blob(container_uuid)
  bson_bytes   = BsonCodec.serialize(contents_doc)
  hash         = BsonCodec.contents_hash(bson_bytes)
  stored_bytes = @format_version == :v2 ? nil : bson_bytes
  columns = %w[UnitID ContainerID ContainmentName TreeConflict ContentsHash]
  values = [unit_blob, parent_blob, containment_name, 0, hash]
  if (conflicts = conflicts_column)
    columns << conflicts
    values << ""
  end
  if contents_column?
    columns << "Contents"
    values << stored_bytes
  end
  placeholders = (["?"] * columns.length).join(", ")
  @db.execute(
    "INSERT INTO Unit (#{columns.join(', ')}) VALUES (#{placeholders})",
    values
  )
  write_v2_unit(uuid, bson_bytes) if @format_version == :v2
  write_stats[:inserted] += 1
  uuid
end

#load_vec_extension! ⇒ Object

Loads the optional sqlite-vec extension into the active connection.



367
368
369
370
371
372
373
374
375
376
377
378
 
# File 'lib/mxrb/io/mpr_file.rb', line 367

def load_vec_extension!
  return true if @vec_loaded

  require "sqlite_vec"
  @db.enable_load_extension(true)
  begin
    SqliteVec.load(@db)
  ensure
    @db.enable_load_extension(false)
  end
  @vec_loaded = true
end

#mendix_version ⇒ Object

── Metadata ─────────────────────────────────────────────────────────



46
47
48
49
50
51
52
 
# File 'lib/mxrb/io/mpr_file.rb', line 46

def mendix_version
  @mendix_version ||= begin
    row = @db.get_first_row("SELECT _ProductVersion FROM _MetaData LIMIT 1") rescue nil
    row ||= @db.get_first_row("SELECT MendixVersion FROM _MetaData LIMIT 1") rescue nil
    row&.first
  end
end

#parse_contents(raw_unit) ⇒ Object

Parse BSON from a raw unit hash.



127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/mxrb/io/mpr_file.rb', line 127

def parse_contents(raw_unit)
  blob = raw_unit["Contents"]
  if (blob.nil? || blob.empty?) && @format_version == :v2
    unit_path = MxunitCodec.path_for(contents_dir, raw_unit.fetch("UnitID"))
    return {} unless File.file?(unit_path)
    return MxunitCodec.read(unit_path)
  end
  return {} if blob.nil? || blob.empty?

  BsonCodec.parse(blob)
end

#project_name ⇒ Object 



66
67
68
69
70
71
72
73
74
75
 
# File 'lib/mxrb/io/mpr_file.rb', line 66

def project_name
  @project_name ||= begin
    # Name lives in the root Unit's BSON ($QualifiedName or Name field)
    root = root_unit
    return nil unless root

    doc = parse_contents(root)
    doc["Name"] || doc["name"] || File.basename(@path, ".mpr")
  end
end

#query(sql, *binds) ⇒ Object 



285
286
287
 
# File 'lib/mxrb/io/mpr_file.rb', line 285

def query(sql, *binds)
  @db.execute(sql, *binds)
end

#read_index_cache(fingerprint) ⇒ Object

Returns the cached index JSON if the fingerprint matches, nil otherwise.



292
293
294
295
296
297
298
299
300
 
# File 'lib/mxrb/io/mpr_file.rb', line 292

def read_index_cache(fingerprint)
  return nil unless tables.include?("_MxrbIndexCache")

  @db.get_first_value(
    "SELECT IndexData FROM _MxrbIndexCache WHERE Fingerprint = ?", [fingerprint]
  )
rescue SQLite3::Exception
  nil
end

#readonly? ⇒ Boolean

Returns:

  • (Boolean) 


29
 
# File 'lib/mxrb/io/mpr_file.rb', line 29

def readonly? = @readonly

#relocate_unit(uuid, container_uuid:, containment_name:) ⇒ Object

Raises:



234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/mxrb/io/mpr_file.rb', line 234

def relocate_unit(uuid, container_uuid:, containment_name:)
  raise ReadOnlyError, "Opened in read-only mode" if @readonly

  @db.execute(
    "UPDATE Unit SET ContainerID = ?, ContainmentName = ? WHERE UnitID = ?",
    [
      BsonCodec.uuid_to_blob(container_uuid),
      containment_name.to_s,
      BsonCodec.uuid_to_blob(uuid)
    ]
  )
end

#restore_from!(backup_path) ⇒ Object

Restores the database from a backup file, replacing the current contents. Closes and reopens the underlying SQLite connection.

Raises:



265
266
267
268
269
270
271
272
273
 
# File 'lib/mxrb/io/mpr_file.rb', line 265

def restore_from!(backup_path)
  raise ReadOnlyError, "Opened in read-only mode" if @readonly

  @db.close
  FileUtils.cp(backup_path, @path)
  @db             = open_db
  @mendix_version = nil
  @format_version = detect_format
end

#root_unit ⇒ Object

The root Unit is the one where UnitID == ContainerID.



80
81
82
83
84
85
86
87
88
 
# File 'lib/mxrb/io/mpr_file.rb', line 80

def root_unit
  @root_unit ||= begin
    row = @db.get_first_row(
      "SELECT #{unit_select_columns} FROM Unit " \
      "WHERE UnitID = ContainerID LIMIT 1"
    )
    row ? raw_to_hash(row) : nil
  end
end

#table_info(name) ⇒ Object 



281
282
283
 
# File 'lib/mxrb/io/mpr_file.rb', line 281

def table_info(name)
  @db.table_info(name)
end

#tables ⇒ Object

── Exploration helpers ───────────────────────────────────────────────



277
278
279
 
# File 'lib/mxrb/io/mpr_file.rb', line 277

def tables
  @db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name").flatten
end

#transaction ⇒ Object 



247
248
249
 
# File 'lib/mxrb/io/mpr_file.rb', line 247

def transaction(&)
  @db.transaction(&)
end

#unit(uuid) ⇒ Object

Single unit by UUID string.



110
111
112
113
114
115
116
117
 
# File 'lib/mxrb/io/mpr_file.rb', line 110

def unit(uuid)
  blob = BsonCodec.uuid_to_blob(uuid)
  row  = @db.get_first_row(
    "SELECT #{unit_select_columns} FROM Unit WHERE UnitID = ?",
    [blob]
  )
  row ? raw_to_hash(row) : nil
end

#units_by_containment(name) ⇒ Object

All units with a given ContainmentName.



91
92
93
94
95
96
97
 
# File 'lib/mxrb/io/mpr_file.rb', line 91

def units_by_containment(name)
  @db.execute(
    "SELECT #{unit_select_columns} FROM Unit " \
    "WHERE ContainmentName = ?",
    [name]
  ).map { raw_to_hash(_1) }
end

#update_unit(uuid, contents_doc) ⇒ Object

Update an existing unit's contents. Recalculates ContentsHash automatically.

Raises:



198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/mxrb/io/mpr_file.rb', line 198

def update_unit(uuid, contents_doc)
  raise ReadOnlyError, "Opened in read-only mode" if @readonly

  blob       = BsonCodec.uuid_to_blob(uuid)
  bson_bytes = BsonCodec.serialize(contents_doc)
  hash       = BsonCodec.contents_hash(bson_bytes)
  current = unit(uuid)
  if current && current['ContentsHash'] == hash
    write_stats[:skipped] += 1
    return false
  end

  if contents_column?
    @db.execute(
      "UPDATE Unit SET Contents = ?, ContentsHash = ? WHERE UnitID = ?",
      [@format_version == :v2 ? nil : bson_bytes, hash, blob]
    )
  else
    @db.execute(
      "UPDATE Unit SET ContentsHash = ? WHERE UnitID = ?",
      [hash, blob]
    )
  end
  write_v2_unit(uuid, bson_bytes) if @format_version == :v2
  write_stats[:updated] += 1
  true
end

#update_version!(version) ⇒ Object 



54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/mxrb/io/mpr_file.rb', line 54

def update_version!(version)
  version_str = version.to_s
  # Try new-style column first, fall back to old-style
  begin
    @db.execute("UPDATE _MetaData SET _ProductVersion = ?, _BuildVersion = ?",
                [version_str, version_str])
  rescue SQLite3::Exception
    @db.execute("UPDATE _MetaData SET MendixVersion = ?", [version_str])
  end
  @mendix_version = version_str
end

#vec_drop_index!(vec_table, meta_table) ⇒ Object 



453
454
455
456
457
 
# File 'lib/mxrb/io/mpr_file.rb', line 453

def vec_drop_index!(vec_table, meta_table)
  ensure_vector_write!
  @db.execute("DROP TABLE IF EXISTS #{vector_identifier(vec_table)}")
  @db.execute("DROP TABLE IF EXISTS #{vector_identifier(meta_table)}")
end

#vec_knn(table, json_vec, limit) ⇒ Object 



439
440
441
442
443
444
445
446
 
# File 'lib/mxrb/io/mpr_file.rb', line 439

def vec_knn(table, json_vec, limit)
  identifier = vector_identifier(table)
  @db.execute(
    "SELECT artifact_id, distance FROM #{identifier} " \
    "WHERE embedding MATCH ? ORDER BY distance LIMIT ?",
    [json_vec, Integer(limit)]
  ).map { { id: _1[0], distance: _1[1] } }
end

#vec_meta(table) ⇒ Object 



415
416
417
418
419
420
421
422
423
424
425
426
427
428
 
# File 'lib/mxrb/io/mpr_file.rb', line 415

def vec_meta(table)
  name = table.to_s
  return nil unless tables.include?(name)

  identifier = vector_identifier(name)
  row = @db.get_first_row(
    "SELECT Backend, Dimension, Fingerprint FROM #{identifier} WHERE ID = 1"
  )
  return nil unless row

  { backend: row[0], dimension: row[1], fingerprint: row[2] }
rescue SQLite3::Exception
  nil
end

#vec_transaction ⇒ Object 



448
449
450
451
 
# File 'lib/mxrb/io/mpr_file.rb', line 448

def vec_transaction(&)
  ensure_vector_write!
  @db.transaction(&)
end

#vec_upsert(table, artifact_id, json_vec) ⇒ Object 



430
431
432
433
434
435
436
437
 
# File 'lib/mxrb/io/mpr_file.rb', line 430

def vec_upsert(table, artifact_id, json_vec)
  ensure_vector_write!
  identifier = vector_identifier(table)
  @db.execute(
    "INSERT INTO #{identifier}(artifact_id, embedding) VALUES (?, ?)",
    [artifact_id, json_vec]
  )
end

#write_architecture_definition(definition) ⇒ Object

Raises:



469
470
471
472
473
474
475
476
477
478
479
480
481
482
 
# File 'lib/mxrb/io/mpr_file.rb', line 469

def write_architecture_definition(definition)
  raise ReadOnlyError, "Opened in read-only mode" if @readonly
  @db.execute(<<~SQL)
    CREATE TABLE IF NOT EXISTS _MxrbArchitecture (
      ID INTEGER PRIMARY KEY CHECK (ID = 1),
      Version INTEGER NOT NULL,
      Definition TEXT NOT NULL
    )
  SQL
  @db.execute(
    "INSERT OR REPLACE INTO _MxrbArchitecture (ID, Version, Definition) VALUES (1, 1, ?)",
    [JSON.generate(definition)]
  )
end

#write_index_cache(fingerprint, json) ⇒ Object

Persists the index JSON keyed by fingerprint. No-op when read-only or on error.



303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
 
# File 'lib/mxrb/io/mpr_file.rb', line 303

def write_index_cache(fingerprint, json)
  return if @readonly || json.nil?

  @db.execute(<<~SQL)
    CREATE TABLE IF NOT EXISTS _MxrbIndexCache (
      Fingerprint TEXT PRIMARY KEY,
      IndexData   TEXT NOT NULL
    )
  SQL
  @db.execute(
    "INSERT INTO _MxrbIndexCache (Fingerprint, IndexData) VALUES (?, ?) " \
    "ON CONFLICT(Fingerprint) DO UPDATE SET IndexData = excluded.IndexData",
    [fingerprint, json]
  )
  @db.execute(
    "DELETE FROM _MxrbIndexCache WHERE Fingerprint <> ?", [fingerprint]
  )
rescue SQLite3::Exception
  nil
end

#write_stats ⇒ Object 



25
26
27
 
# File 'lib/mxrb/io/mpr_file.rb', line 25

def write_stats
  @write_stats ||= { inserted: 0, updated: 0, skipped: 0, deleted: 0 }
end

#write_vec_meta!(table, backend, dimension, fingerprint) ⇒ Object 



405
406
407
408
409
410
411
412
413
 
# File 'lib/mxrb/io/mpr_file.rb', line 405

def write_vec_meta!(table, backend, dimension, fingerprint)
  ensure_vector_write!
  identifier = vector_identifier(table)
  @db.execute(
    "INSERT OR REPLACE INTO #{identifier} " \
    "(ID, Backend, Dimension, Fingerprint) VALUES (1, ?, ?, ?)",
    [backend, dimension, fingerprint]
  )
end