Module: Opencdd::Parcel

Defined in:
lib/opencdd/parcel.rb,
lib/opencdd/parcel/sheet.rb,
lib/opencdd/parcel/writer.rb,
lib/opencdd/parcel/metadata.rb,
lib/opencdd/parcel/selector.rb,
lib/opencdd/parcel/workbook.rb,
lib/opencdd/parcel/csv_reader.rb,
lib/opencdd/parcel/csv_writer.rb,
lib/opencdd/parcel/sheet_schema.rb,
lib/opencdd/parcel/sheet_emitter.rb,
lib/opencdd/parcel/entity_manifest.rb,
lib/opencdd/parcel/flat_dir_reader.rb,
lib/opencdd/parcel/layout_detector.rb,
lib/opencdd/parcel/scrape_verifier.rb,
lib/opencdd/parcel/workbook_reader.rb,
lib/opencdd/parcel/referenced_irdis.rb,
lib/opencdd/parcel/sharded_dir_reader.rb

Defined Under Namespace

Modules: CsvReader, CsvWriter, LayoutDetector Classes: EntityManifest, FlatDirReader, Metadata, ReferencedIrdis, ScrapeVerifier, Selector, ShardedDirReader, Sheet, SheetEmitter, SheetSchema, Workbook, WorkbookReader, Writer

Constant Summary collapse

META_CLASS_TYPES = 
Opencdd::MetaClasses::TYPE_BY_META_CLASS
TYPE_TO_META_CLASS = 
META_CLASS_TYPES.invert.freeze

Class Method Summary collapse

Class Method Details

.aggregate(*paths) ⇒ Object

Aggregate multiple Parcel-shaped paths into a single Database. The paths array may contain .xlsx files, .xls files, flat .xls directories, or sharded per-class subdirs. Each is read polymorphically via Opencdd::Reader.detect and merged into a single target database.

Returns the populated Opencdd::Database.



34
35
36
37
38
 
# File 'lib/opencdd/parcel.rb', line 34

def aggregate(*paths)
  paths.flatten
    .map { |p| Opencdd::Database.load(p) }
    .reduce(Opencdd::Database.new) { |acc, db| acc.merge(db) }
end

.build_partition(_database, entities) ⇒ Object 



149
150
151
152
153
154
 
# File 'lib/opencdd/parcel.rb', line 149

def build_partition(_database, entities)
  db = Opencdd::Database.new
  entities.each { |e| db.add_entity(deep_copy_entity(e)) }
  db.finalize!
  db
end

.collect_root_subtree(database, root) ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/opencdd/parcel.rb', line 108

def collect_root_subtree(database, root)
  collected = [root]
  queue = [root]
  seen = { root.irdi => true }
  until queue.empty?
    current = queue.shift
    children = current.children
    children.each do |c|
      next if seen[c.irdi]
      seen[c.irdi] = true
      collected << c
      queue << c
    end
  end
  collected
end

.deep_copy_entity(entity) ⇒ Object

Returns a detached copy of entity — same irdi, properties, schema, meta_class_irdi, but no @database back-reference and empty Klass-specific state (+@children+, @declared_property_irdis).

The destination partition Database reattaches the entity and rebuilds class-tree state via add_entity and finalize!.

We can't use Marshal here because Entity indirectly references its source Database, which uses Hash.new { ... } (a default proc) for its indexes — Marshal refuses to dump hashes with default procs.



166
167
168
169
170
171
172
173
 
# File 'lib/opencdd/parcel.rb', line 166

def deep_copy_entity(entity)
  entity.class.new(
    irdi: entity.irdi,
    properties: entity.properties.dup,
    schema: entity.schema,
    meta_class_irdi: entity.meta_class_irdi,
  )
end

.lift_for(database, classes) ⇒ Object

Lifts cross-type dependencies for the given set of classes: declared properties, value_lists those properties reference, units those properties use, and relations where the class is in domain or codomain. This makes a per-tree partition self-sufficient when written out.



130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/opencdd/parcel.rb', line 130

def lift_for(database, classes)
  entities = classes.dup
  classes.each do |klass|
    database.properties_of(klass).each { |p| entities << p unless entities.include?(p) }
    database.relations_for(domain: klass.irdi).each { |r| entities << r unless entities.include?(r) }
    database.relations_for(codomain: klass.irdi).each { |r| entities << r unless entities.include?(r) }
    database.properties_of(klass).each do |prop|
      next unless prop.is_a?(Opencdd::Property)
      if prop.unit_irdi
        unit = database.find(prop.unit_irdi)
        entities << unit if unit.is_a?(Opencdd::Unit) && !entities.include?(unit)
      end
      vl = database.value_list_of(prop)
      entities << vl if vl && !entities.include?(vl)
    end
  end
  entities.uniq
end

.split(database, by:, lift_dependencies: true) ⇒ Object

Split database into multiple sub-databases partitioned along by:

:entity_type — one partition per entity type
(the legacy .xls layout, one file per type).
:root_class — one partition per root class
(full tree per parcel, lifted cross-type deps).
:each_class — one partition per class
(the class + its declared properties / value_lists /
relations / units).
callable   — partition key is +callable.call(entity)+;
the result is a +Hash{ key => Database }+.

Returns Hash{ key => Opencdd::Database }.



54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/opencdd/parcel.rb', line 54

def split(database, by:, lift_dependencies: true)
  partitions = case by
               when :entity_type then split_by_type(database)
               when :root_class  then split_by_root_class(database, lift_dependencies: lift_dependencies)
               when :each_class  then split_by_each_class(database, lift_dependencies: lift_dependencies)
               when Proc         then split_by_callable(database, by)
               else
                 raise ArgumentError, "split `by:` must be :entity_type, :root_class, :each_class, or a Proc"
               end
  partitions
end

.split_by_callable(database, callable) ⇒ Object 



102
103
104
105
106
 
# File 'lib/opencdd/parcel.rb', line 102

def split_by_callable(database, callable)
  database.entities.group_by { |e| callable.call(e) }.each_with_object({}) do |(key, entities), h|
    h[key] = build_partition(database, entities)
  end
end

.split_by_each_class(database, lift_dependencies: true) ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/opencdd/parcel.rb', line 81

def split_by_each_class(database, lift_dependencies: true)
  database.classes.each_with_object({}) do |klass, h|
    entities = [klass]
    if lift_dependencies
      klass_props = database.properties_of(klass)
      entities.concat(klass_props)
      entities.concat(database.relations_for(domain: klass.irdi))
      klass_props.each do |prop|
        next unless prop.is_a?(Opencdd::Property)
        if prop.unit_irdi
          unit = database.find(prop.unit_irdi)
          entities << unit if unit.is_a?(Opencdd::Unit)
        end
        vl = database.value_list_of(prop)
        entities << vl if vl
      end
    end
    h[klass.code || klass.irdi.to_s] = build_partition(database, entities)
  end
end

.split_by_root_class(database, lift_dependencies: true) ⇒ Object 



72
73
74
75
76
77
78
79
 
# File 'lib/opencdd/parcel.rb', line 72

def split_by_root_class(database, lift_dependencies: true)
  roots = database.root_classes
  roots.each_with_object({}) do |root, h|
    tree = collect_root_subtree(database, root)
    entities = lift_dependencies ? lift_for(database, tree) : tree
    h[root.code || root.irdi.to_s] = build_partition(database, entities)
  end
end

.split_by_type(database) ⇒ Object 



66
67
68
69
70
 
# File 'lib/opencdd/parcel.rb', line 66

def split_by_type(database)
  database.entities.group_by(&:type).each_with_object({}) do |(type, entities), h|
    h[type] = build_partition(database, entities)
  end
end