Module: Secryst::IMF

Defined in:
lib/secryst/imf.rb

Overview

Interscript Model Format v1 — the byte-level runtime contract shared with the Python (interscript-ml) and TypeScript (npm: secryst) runtimes. Token ids follow the canonical ByT5 table: byte b -> b+3, trailing EOS; pad=0, unk=2. Ids are NOT raw byte values.

Defined Under Namespace

Classes: FormatError, RegistryError

Constant Summary collapse

BYTE_OFFSET = 
3
PAD_ID = 
0
EOS_ID = 
1
UNK_ID = 
2
DEFAULT_INDEX_URL = 
"https://raw.githubusercontent.com/interscript/interscript-ml/main/models.yaml"

Class Method Summary collapse

Class Method Details

.cache_dir ⇒ Object 



74
75
76
 
# File 'lib/secryst/imf.rb', line 74

def cache_dir
  ENV["SECRYST_CACHE"] || File.join(Dir.home, ".cache", "secryst")
end

.decode(token_ids) ⇒ Object 



28
29
30
31
32
33
34
35
36
 
# File 'lib/secryst/imf.rb', line 28

def decode(token_ids)
  out = +""
  token_ids.each do |token|
    break if token == EOS_ID
    next if token == PAD_ID || token == UNK_ID
    out << ((token - BYTE_OFFSET) % 256).chr
  end
  out.force_encoding(Encoding::UTF_8)
end

.encode(text) ⇒ Object 



24
25
26
 
# File 'lib/secryst/imf.rb', line 24

def encode(text)
  text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID]
end

.manifest(zip_path) ⇒ Object 



38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/secryst/imf.rb', line 38

def manifest(zip_path)
  Zip::File.open(zip_path) do |zf|
    raise FormatError, "missing metadata.yaml" unless zf.find_entry("metadata.yaml")
    meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false)
    raise FormatError, "unsupported format: #{meta["format"].inspect}" if meta["format"] != "imf-v1"
    if meta["tokenizer"] != "bytes"
      raise FormatError, "tokenizer #{meta["tokenizer"].inspect}: this runtime is byte-level only"
    end
    %w[encoder.onnx decoder.onnx].each do |required|
      raise FormatError, "missing #{required}" unless zf.find_entry(required)
    end
    meta
  end
end

.resolve(model_id, index_url: nil) ⇒ Object

models.yaml resolution: cache hit (re-verified), or download -> verify whole-file sha256 -> atomic install into the cache. Entries with parts (GitHub's 2 GiB per-asset cap) are streamed in order, each part sha256-verified as it lands, then the assembled file is checked against the whole-file sha256.

Raises:



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/secryst/imf.rb', line 83

def resolve(model_id, index_url: nil)
  source = index_url || ENV["SECRYST_INDEX"] || DEFAULT_INDEX_URL
  entries = load_index(source)
  entry = entries[model_id]
  raise RegistryError, "unknown model id #{model_id.inspect} (known: #{entries.keys.sort})" unless entry

  target = File.join(cache_dir, "models", model_id, entry["filename"])
  if File.file?(target) && Digest::SHA256.file(target).hexdigest == entry["sha256"]
    return target
  end

  FileUtils.mkdir_p(File.dirname(target))
  tmp = target + ".part.#{Process.pid}"
  if entry["parts"]
    download_parts(entry, tmp)
  else
    channel = entry["url"]
    if channel.start_with?("file://")
      FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp)
    else
      URI.open(channel) { |remote| IO.copy_stream(remote, tmp) }
    end
  end
  actual = Digest::SHA256.file(tmp).hexdigest
  unless actual == entry["sha256"]
    File.delete(tmp)
    raise RegistryError, "downloaded #{entry["filename"]} sha256 mismatch: got #{actual}, index says #{entry["sha256"]}"
  end
  File.rename(tmp, target)
  target
end

.verify_and_read(zip_path) ⇒ Object

Reads every .onnx member after verifying its sha256 against the manifest — corrupt zips fail loudly, before any session loads.



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/secryst/imf.rb', line 55

def verify_and_read(zip_path)
  meta = manifest(zip_path)
  sha = meta.fetch("sha256", {})
  graphs = {}
  Zip::File.open(zip_path) do |zf|
    zf.entries.select { |e| e.name.end_with?(".onnx") }.each do |entry|
      recorded = sha[entry.name]
      raise FormatError, "#{entry.name} is not covered by metadata sha256" unless recorded
      bytes = entry.get_input_stream.read
      actual = Digest::SHA256.hexdigest(bytes)
      if actual != recorded
        raise FormatError, "#{entry.name} sha256 mismatch: zip has #{actual}, metadata says #{recorded}"
      end
      graphs[entry.name] = bytes
    end
  end
  graphs
end