Integrating Rust fake-rs into FakeDataDSL Ruby Gem
A step-by-step guide to adding a high-performance Rust native extension to FakeDataDSL using fake-rs and Magnus (Ruby-Rust bindings).
Table of Contents
- Prerequisites
- Project Structure
- Setting Up the Rust Extension
- Writing the Rust Code
- Building the Extension
- Integrating with Ruby
- Testing
- Packaging the Gem
- Performance Benchmarks
- Troubleshooting
1. Prerequisites
Install Rust
# macOS/Linux
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
# Verify installation
rustc --version # Should show 1.70+
cargo --version
Install rb-sys (Ruby-Rust bridge)
gem install rb_sys
System Dependencies
# macOS
xcode-select --install
# Ubuntu/Debian
sudo apt-get install build-essential libclang-dev
# Fedora/RHEL
sudo dnf install clang-devel
2. Project Structure
Add these files/directories to your gem:
fake_data_dsl/
├── ext/
│ └── fake_data_dsl_native/ # Rust extension
│ ├── Cargo.toml # Rust dependencies
│ ├── extconf.rb # Ruby extension config
│ └── src/
│ ├── lib.rs # Main entry point
│ ├── types.rs # Type generators
│ ├── generator.rs # Generation engine
│ └── schema.rs # Schema parsing
├── lib/
│ └── fake_data_dsl/
│ ├── native_engine.rb # Ruby wrapper
│ └── ...
├── fake_data_dsl.gemspec
└── Rakefile
3. Setting Up the Rust Extension
3.1 Create the Extension Directory
mkdir -p ext/fake_data_dsl_native/src
3.2 Create ext/fake_data_dsl_native/Cargo.toml
[package]
name = "fake_data_dsl_native"
version = "0.1.0"
edition = "2021"
authors = ["Your Name <your.email@example.com>"]
license = "MIT"
publish = false
[lib]
crate-type = ["cdylib"]
[dependencies]
# Ruby bindings
magnus = "0.6"
rb-sys = "0.9"
# Fake data generation (like Ruby's Faker)
fake = { version = "4", features = [
"derive",
"chrono",
"uuid",
"random_color",
"http"
]}
# Random number generation
rand = "0.8"
rand_chacha = "0.3"
# Parallel processing (optional, for multi-core)
rayon = "1.8"
# JSON handling
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
# Date/time
chrono = "0.4"
# UUID
uuid = { version = "1.0", features = ["v4", "v1", "serde"] }
[profile.release]
opt-level = 3
lto = true
codegen-units = 1
3.3 Create ext/fake_data_dsl_native/extconf.rb
# frozen_string_literal: true
require "mkmf"
require "rb_sys/mkmf"
create_rust_makefile("fake_data_dsl_native") do |r|
# Release mode for best performance
r.profile = ENV.fetch("RB_SYS_CARGO_PROFILE", :release).to_sym
# Extra features
r.extra_cargo_args = ["--features", "parallel"]
end
3.4 Update Rakefile
require "bundler/gem_tasks"
require "rspec/core/rake_task"
require "rb_sys/extensiontask"
RSpec::Core::RakeTask.new(:spec)
# Native extension build task
RbSys::ExtensionTask.new("fake_data_dsl_native", GEMSPEC) do |ext|
ext.lib_dir = "lib/fake_data_dsl"
end
task build: :compile
task default: [:compile, :spec]
3.5 Update fake_data_dsl.gemspec
Gem::Specification.new do |spec|
spec.name = "fake_data_dsl"
spec.version = FakeDataDSL::VERSION
# ... other fields ...
# Add extension
spec.extensions = ["ext/fake_data_dsl_native/extconf.rb"]
# Dependencies for building
spec.add_dependency "rb_sys", "~> 0.9"
# Include Rust files
spec.files += Dir["ext/**/*.{rs,toml,rb}"]
# Metadata
spec.["rubygems_mfa_required"] = "true"
end
4. Writing the Rust Code
4.1 Main Entry Point: src/lib.rs
use magnus::{define_module, function, prelude::*, Error, RArray, Ruby, Value};
mod generator;
mod schema;
mod types;
/// Generate records and return as Ruby Array of Hashes
fn generate(
ruby: &Ruby,
schema_spec: RArray,
count: usize,
seed: Option<u64>,
locale: Option<String>,
) -> Result<RArray, Error> {
let generator = generator::NativeGenerator::from_ruby(schema_spec, seed, locale)?;
generator.generate_to_ruby(ruby, count)
}
/// Generate records directly to a file (most efficient)
fn generate_to_file(
schema_spec: RArray,
count: usize,
path: String,
seed: Option<u64>,
locale: Option<String>,
format: Option<String>,
) -> Result<usize, Error> {
let generator = generator::NativeGenerator::from_ruby(schema_spec, seed, locale)?;
let fmt = format.as_deref().unwrap_or("jsonl");
generator.generate_to_file(count, &path, fmt)
.map_err(|e| Error::new(ruby_exception(), e.to_string()))
}
/// Generate a single record (for testing)
fn generate_one(
ruby: &Ruby,
schema_spec: RArray,
seed: Option<u64>,
locale: Option<String>,
) -> Result<Value, Error> {
let generator = generator::NativeGenerator::from_ruby(schema_spec, seed, locale)?;
generator.generate_one_to_ruby(ruby)
}
/// Get list of available types
fn available_types() -> Vec<String> {
types::AVAILABLE_TYPES.iter().map(|s| s.to_string()).collect()
}
/// Get list of available locales
fn available_locales() -> Vec<String> {
vec![
"en", "fr_fr", "de_de", "zh_cn", "zh_tw",
"ja_jp", "pt_br", "it_it", "nl_nl", "ar_sa"
].iter().map(|s| s.to_string()).collect()
}
/// Get native extension version
fn version() -> String {
env!("CARGO_PKG_VERSION").to_string()
}
/// Check if extension is loaded
fn loaded() -> bool {
true
}
fn ruby_exception() -> magnus::ExceptionClass {
magnus::exception::runtime_error()
}
#[magnus::init]
fn init(ruby: &Ruby) -> Result<(), Error> {
let module = ruby.define_module("FakeDataDSL")?;
let native = module.define_module("Native")?;
native.define_singleton_method("generate", function!(generate, 4))?;
native.define_singleton_method("generate_to_file", function!(generate_to_file, 6))?;
native.define_singleton_method("generate_one", function!(generate_one, 3))?;
native.define_singleton_method("available_types", function!(available_types, 0))?;
native.define_singleton_method("available_locales", function!(available_locales, 0))?;
native.define_singleton_method("version", function!(version, 0))?;
native.define_singleton_method("loaded?", function!(loaded, 0))?;
Ok(())
}
4.2 Type Generators: src/types.rs
use fake::faker::address::raw::*;
use fake::faker::company::raw::*;
use fake::faker::internet::raw::*;
use fake::faker::lorem::raw::*;
use fake::faker::name::raw::*;
use fake::faker::phone_number::raw::*;
use fake::locales::*;
use fake::Fake;
use rand::Rng;
use serde_json::{json, Value};
use uuid::Uuid;
/// List of all supported type names
pub static AVAILABLE_TYPES: &[&str] = &[
// Personal
"name", "first_name", "last_name", "full_name", "title",
// Internet
"email", "username", "password", "url", "ip", "ipv4", "ipv6",
"mac_address", "user_agent", "domain",
// Address
"city", "country", "country_code", "street", "street_address",
"postal_code", "zip_code", "latitude", "longitude", "state",
// Company
"company_name", "company", "catch_phrase", "buzzword", "industry",
// Text
"word", "words", "sentence", "sentences", "paragraph", "paragraphs", "text",
// Phone
"phone", "phone_number", "cell_phone",
// Date/Time
"date", "time", "datetime", "timestamp", "past_date", "future_date",
// Identifiers
"uuid", "uuid_v4", "uuid_v1", "id_sequence",
// Finance
"credit_card", "currency_code", "bic",
// Primitives
"number", "integer", "float", "boolean", "bool",
// Color
"hex_color", "rgb_color",
// Special
"enum", "const", "array",
];
/// Locale enum matching fake-rs locales
#[derive(Clone, Copy, Debug)]
pub enum Locale {
EN,
FR,
DE,
ZH_CN,
ZH_TW,
JA,
PT_BR,
IT,
NL,
AR,
}
impl Locale {
pub fn from_string(s: &str) -> Self {
match s.to_lowercase().as_str() {
"fr" | "fr_fr" | "french" => Locale::FR,
"de" | "de_de" | "german" => Locale::DE,
"zh_cn" | "chinese" | "chinese_simplified" => Locale::ZH_CN,
"zh_tw" | "chinese_traditional" => Locale::ZH_TW,
"ja" | "ja_jp" | "japanese" => Locale::JA,
"pt_br" | "portuguese" | "brazilian" => Locale::PT_BR,
"it" | "it_it" | "italian" => Locale::IT,
"nl" | "nl_nl" | "dutch" => Locale::NL,
"ar" | "ar_sa" | "arabic" => Locale::AR,
_ => Locale::EN,
}
}
}
/// Generate a value for a given type
pub fn generate_value<R: Rng>(
rng: &mut R,
type_name: &str,
args: &Value,
locale: Locale,
index: usize,
) -> Value {
match type_name {
// ========== Personal Names ==========
"name" | "full_name" => match locale {
Locale::EN => json!(Name(EN).fake_with_rng::<String, _>(rng)),
Locale::FR => json!(Name(FR_FR).fake_with_rng::<String, _>(rng)),
Locale::DE => json!(Name(DE_DE).fake_with_rng::<String, _>(rng)),
Locale::ZH_CN => json!(Name(ZH_CN).fake_with_rng::<String, _>(rng)),
Locale::ZH_TW => json!(Name(ZH_TW).fake_with_rng::<String, _>(rng)),
Locale::JA => json!(Name(JA_JP).fake_with_rng::<String, _>(rng)),
Locale::PT_BR => json!(Name(PT_BR).fake_with_rng::<String, _>(rng)),
Locale::IT => json!(Name(IT_IT).fake_with_rng::<String, _>(rng)),
Locale::NL => json!(Name(NL_NL).fake_with_rng::<String, _>(rng)),
Locale::AR => json!(Name(AR_SA).fake_with_rng::<String, _>(rng)),
},
"first_name" => match locale {
Locale::EN => json!(FirstName(EN).fake_with_rng::<String, _>(rng)),
Locale::FR => json!(FirstName(FR_FR).fake_with_rng::<String, _>(rng)),
Locale::DE => json!(FirstName(DE_DE).fake_with_rng::<String, _>(rng)),
Locale::ZH_CN => json!(FirstName(ZH_CN).fake_with_rng::<String, _>(rng)),
Locale::ZH_TW => json!(FirstName(ZH_TW).fake_with_rng::<String, _>(rng)),
Locale::JA => json!(FirstName(JA_JP).fake_with_rng::<String, _>(rng)),
_ => json!(FirstName(EN).fake_with_rng::<String, _>(rng)),
},
"last_name" => match locale {
Locale::EN => json!(LastName(EN).fake_with_rng::<String, _>(rng)),
Locale::FR => json!(LastName(FR_FR).fake_with_rng::<String, _>(rng)),
Locale::DE => json!(LastName(DE_DE).fake_with_rng::<String, _>(rng)),
Locale::ZH_CN => json!(LastName(ZH_CN).fake_with_rng::<String, _>(rng)),
Locale::ZH_TW => json!(LastName(ZH_TW).fake_with_rng::<String, _>(rng)),
Locale::JA => json!(LastName(JA_JP).fake_with_rng::<String, _>(rng)),
_ => json!(LastName(EN).fake_with_rng::<String, _>(rng)),
},
"title" => json!(Title(EN).fake_with_rng::<String, _>(rng)),
// ========== Internet ==========
"email" => {
let base: String = FreeEmail(EN).fake_with_rng(rng);
// Make email unique by adding index
let parts: Vec<&str> = base.splitn(2, '@').collect();
if parts.len() == 2 {
json!(format!("{}{}@{}", parts[0], index, parts[1]))
} else {
json!(base)
}
}
"username" => json!(Username(EN).fake_with_rng::<String, _>(rng)),
"password" => {
let min = args.get("min").and_then(|v| v.as_u64()).unwrap_or(8) as usize;
let max = args.get("max").and_then(|v| v.as_u64()).unwrap_or(20) as usize;
json!(Password(EN, min..max).fake_with_rng::<String, _>(rng))
}
"url" => {
let domain: String = DomainSuffix(EN).fake_with_rng(rng);
json!(format!("https://www.{}.{}",
Word(EN).fake_with_rng::<String, _>(rng).to_lowercase(),
domain
))
}
"ip" | "ipv4" => json!(IPv4(EN).fake_with_rng::<String, _>(rng)),
"ipv6" => json!(IPv6(EN).fake_with_rng::<String, _>(rng)),
"mac_address" => json!(MACAddress(EN).fake_with_rng::<String, _>(rng)),
"user_agent" => json!(UserAgent(EN).fake_with_rng::<String, _>(rng)),
"domain" => json!(DomainSuffix(EN).fake_with_rng::<String, _>(rng)),
// ========== Address ==========
"city" => match locale {
Locale::EN => json!(CityName(EN).fake_with_rng::<String, _>(rng)),
Locale::FR => json!(CityName(FR_FR).fake_with_rng::<String, _>(rng)),
Locale::DE => json!(CityName(DE_DE).fake_with_rng::<String, _>(rng)),
_ => json!(CityName(EN).fake_with_rng::<String, _>(rng)),
},
"country" => json!(CountryName(EN).fake_with_rng::<String, _>(rng)),
"country_code" => json!(CountryCode(EN).fake_with_rng::<String, _>(rng)),
"street" | "street_address" => match locale {
Locale::EN => json!(StreetName(EN).fake_with_rng::<String, _>(rng)),
Locale::FR => json!(StreetName(FR_FR).fake_with_rng::<String, _>(rng)),
_ => json!(StreetName(EN).fake_with_rng::<String, _>(rng)),
},
"postal_code" | "zip_code" => json!(ZipCode(EN).fake_with_rng::<String, _>(rng)),
"latitude" => {
let lat: f64 = Latitude(EN).fake_with_rng(rng);
json!(lat)
}
"longitude" => {
let lng: f64 = Longitude(EN).fake_with_rng(rng);
json!(lng)
}
"state" => json!(StateName(EN).fake_with_rng::<String, _>(rng)),
// ========== Company ==========
"company" | "company_name" => json!(CompanyName(EN).fake_with_rng::<String, _>(rng)),
"catch_phrase" => json!(CatchPhrase(EN).fake_with_rng::<String, _>(rng)),
"buzzword" => json!(Buzzword(EN).fake_with_rng::<String, _>(rng)),
"industry" => json!(Industry(EN).fake_with_rng::<String, _>(rng)),
// ========== Text ==========
"word" => json!(Word(EN).fake_with_rng::<String, _>(rng)),
"words" => {
let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(5) as usize;
let words: Vec<String> = Words(EN, count..count + 1).fake_with_rng(rng);
json!(words.join(" "))
}
"sentence" => {
let words = args.get("words").and_then(|v| v.as_u64()).unwrap_or(10) as usize;
json!(Sentence(EN, words..words + 5).fake_with_rng::<String, _>(rng))
}
"sentences" => {
let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(3) as usize;
json!(Sentences(EN, count..count + 2).fake_with_rng::<Vec<String>, _>(rng).join(" "))
}
"paragraph" => {
let sentences = args.get("sentences").and_then(|v| v.as_u64()).unwrap_or(5) as usize;
json!(Paragraph(EN, sentences..sentences + 3).fake_with_rng::<String, _>(rng))
}
"paragraphs" => {
let count = args.get("count").and_then(|v| v.as_u64()).unwrap_or(3) as usize;
json!(Paragraphs(EN, count..count + 2).fake_with_rng::<Vec<String>, _>(rng).join("\n\n"))
}
"text" => {
let min = args.get("min").and_then(|v| v.as_u64()).unwrap_or(10) as usize;
let max = args.get("max").and_then(|v| v.as_u64()).unwrap_or(100) as usize;
let len = rng.gen_range(min..=max);
let text: String = (0..len)
.map(|_| rng.gen_range(b'a'..=b'z') as char)
.collect();
json!(text)
}
// ========== Phone ==========
"phone" | "phone_number" => json!(PhoneNumber(EN).fake_with_rng::<String, _>(rng)),
"cell_phone" => json!(CellNumber(EN).fake_with_rng::<String, _>(rng)),
// ========== Date/Time ==========
"date" => {
let days_ago = rng.gen_range(0i64..365);
let date = chrono::Utc::now() - chrono::Duration::days(days_ago);
json!(date.format("%Y-%m-%d").to_string())
}
"time" => {
let hour = rng.gen_range(0..24);
let minute = rng.gen_range(0..60);
let second = rng.gen_range(0..60);
json!(format!("{:02}:{:02}:{:02}", hour, minute, second))
}
"datetime" | "timestamp" => {
let days_ago = rng.gen_range(0i64..365);
let dt = chrono::Utc::now() - chrono::Duration::days(days_ago);
json!(dt.to_rfc3339())
}
"past_date" => {
let days_ago = rng.gen_range(1i64..3650);
let date = chrono::Utc::now() - chrono::Duration::days(days_ago);
json!(date.format("%Y-%m-%d").to_string())
}
"future_date" => {
let days_ahead = rng.gen_range(1i64..365);
let date = chrono::Utc::now() + chrono::Duration::days(days_ahead);
json!(date.format("%Y-%m-%d").to_string())
}
// ========== Identifiers ==========
"uuid" | "uuid_v4" => json!(Uuid::new_v4().to_string()),
"uuid_v1" => {
// Sequential UUID-like format
json!(format!(
"{:08x}-{:04x}-1000-8000-{:012x}",
(index >> 32) as u32,
(index >> 16) as u16,
index as u64
))
}
"id_sequence" => {
let start = args.get("start").and_then(|v| v.as_u64()).unwrap_or(1);
json!(start + index as u64)
}
// ========== Primitives ==========
"number" | "integer" => {
let min = args.get("min").and_then(|v| v.as_i64()).unwrap_or(0);
let max = args.get("max").and_then(|v| v.as_i64()).unwrap_or(100);
json!(rng.gen_range(min..=max))
}
"float" => {
let min = args.get("min").and_then(|v| v.as_f64()).unwrap_or(0.0);
let max = args.get("max").and_then(|v| v.as_f64()).unwrap_or(100.0);
let precision = args.get("precision").and_then(|v| v.as_u64()).unwrap_or(2) as i32;
let val = rng.gen_range(min..=max);
let multiplier = 10_f64.powi(precision);
json!((val * multiplier).round() / multiplier)
}
"boolean" | "bool" => {
let prob = args
.get("probability")
.or_else(|| args.get("true_probability"))
.and_then(|v| v.as_f64())
.unwrap_or(0.5);
json!(rng.gen_bool(prob))
}
// ========== Color ==========
"hex_color" => {
use fake::faker::color::raw::HexColor;
json!(HexColor(EN).fake_with_rng::<String, _>(rng))
}
"rgb_color" => {
let r = rng.gen_range(0..256);
let g = rng.gen_range(0..256);
let b = rng.gen_range(0..256);
json!(format!("rgb({}, {}, {})", r, g, b))
}
// ========== Special Types ==========
"enum" => {
if let Some(values) = args.get("values").and_then(|v| v.as_array()) {
if !values.is_empty() {
return values[rng.gen_range(0..values.len())].clone();
}
}
Value::Null
}
"const" => args.get("value").cloned().unwrap_or(Value::Null),
"array" => {
let item_type = args.get("type").and_then(|v| v.as_str()).unwrap_or("word");
let min = args.get("min").and_then(|v| v.as_u64()).unwrap_or(1) as usize;
let max = args.get("max").and_then(|v| v.as_u64()).unwrap_or(5) as usize;
let count = rng.gen_range(min..=max);
let items: Vec<Value> = (0..count)
.map(|i| generate_value(rng, item_type, &json!({}), locale, index + i))
.collect();
json!(items)
}
// Unknown type
_ => Value::Null,
}
}
4.3 Generator Engine: src/generator.rs
use magnus::{Error, RArray, RHash, Ruby, Symbol, Value};
use rand::SeedableRng;
use rand_chacha::ChaCha8Rng;
use serde_json::Map;
use std::fs::File;
use std::io::{BufWriter, Write};
use crate::types::{generate_value, Locale};
/// Field specification parsed from Ruby
#[derive(Clone)]
pub struct FieldSpec {
pub name: String,
pub type_name: String,
pub args: serde_json::Value,
pub optional: bool,
pub nullable: bool,
}
/// Native generator engine
pub struct NativeGenerator {
fields: Vec<FieldSpec>,
seed: Option<u64>,
locale: Locale,
}
impl NativeGenerator {
/// Create generator from Ruby schema specification
///
/// Expected format: Array of [name, type_name, args_json, optional, nullable]
pub fn from_ruby(schema_spec: RArray, seed: Option<u64>, locale: Option<String>) -> Result<Self, Error> {
let mut fields = Vec::new();
for item in schema_spec.each() {
let item = item?;
let field_array: RArray = item.try_convert()?;
let name: String = field_array.entry::<String>(0)?;
let type_name: String = field_array.entry::<String>(1)?;
let args_json: String = field_array.entry::<String>(2)?;
let optional: bool = field_array.entry::<bool>(3).unwrap_or(false);
let nullable: bool = field_array.entry::<bool>(4).unwrap_or(false);
let args: serde_json::Value = serde_json::from_str(&args_json)
.unwrap_or(serde_json::Value::Object(Map::new()));
fields.push(FieldSpec {
name,
type_name,
args,
optional,
nullable,
});
}
let loc = locale.map(|s| Locale::from_string(&s)).unwrap_or(Locale::EN);
Ok(Self {
fields,
seed,
locale: loc,
})
}
/// Generate records and return as Ruby Array of Hashes
pub fn generate_to_ruby(&self, ruby: &Ruby, count: usize) -> Result<RArray, Error> {
let result = ruby.ary_new_capa(count);
let mut rng = self.create_rng(0);
for i in 0..count {
let record = self.generate_record(&mut rng, i)?;
let hash = self.value_to_ruby_hash(ruby, &record)?;
result.push(hash)?;
}
Ok(result)
}
/// Generate a single record to Ruby Hash
pub fn generate_one_to_ruby(&self, ruby: &Ruby) -> Result<Value, Error> {
let mut rng = self.create_rng(0);
let record = self.generate_record(&mut rng, 0)?;
self.value_to_ruby_hash(ruby, &record)
}
/// Generate records directly to file (most efficient)
pub fn generate_to_file(&self, count: usize, path: &str, format: &str) -> std::io::Result<usize> {
let file = File::create(path)?;
let mut writer = BufWriter::with_capacity(64 * 1024 * 1024, file); // 64MB buffer
let mut rng = self.create_rng(0);
match format {
"csv" => {
// Write header
let headers: Vec<&str> = self.fields.iter().map(|f| f.name.as_str()).collect();
writeln!(writer, "{}", headers.join(","))?;
// Write rows
for i in 0..count {
let record = self.generate_record(&mut rng, i)?;
let values: Vec<String> = self.fields.iter().map(|f| {
match record.get(&f.name) {
Some(serde_json::Value::String(s)) => {
if s.contains(',') || s.contains('"') || s.contains('\n') {
format!("\"{}\"", s.replace('"', "\"\""))
} else {
s.clone()
}
}
Some(v) => v.to_string(),
None => String::new(),
}
}).collect();
writeln!(writer, "{}", values.join(","))?;
}
}
"json" => {
// Write as JSON array
write!(writer, "[")?;
for i in 0..count {
if i > 0 {
write!(writer, ",")?;
}
let record = self.generate_record(&mut rng, i)?;
write!(writer, "{}", serde_json::to_string(&record).unwrap())?;
}
writeln!(writer, "]")?;
}
_ => {
// Default: JSONL (one JSON object per line)
for i in 0..count {
let record = self.generate_record(&mut rng, i)?;
writeln!(writer, "{}", serde_json::to_string(&record).unwrap())?;
}
}
}
writer.flush()?;
Ok(count)
}
/// Generate a single record as JSON object
fn generate_record(&self, rng: &mut ChaCha8Rng, index: usize) -> std::io::Result<Map<String, serde_json::Value>> {
let mut record = Map::new();
for field in &self.fields {
// Handle optional fields
if field.optional && rng.gen_bool(0.2) {
continue; // Skip 20% of optional fields
}
// Handle nullable fields
if field.nullable && rng.gen_bool(0.1) {
record.insert(field.name.clone(), serde_json::Value::Null);
continue;
}
let value = generate_value(rng, &field.type_name, &field.args, self.locale, index);
record.insert(field.name.clone(), value);
}
Ok(record)
}
/// Create RNG from seed
fn create_rng(&self, offset: u64) -> ChaCha8Rng {
match self.seed {
Some(seed) => ChaCha8Rng::seed_from_u64(seed.wrapping_add(offset)),
None => ChaCha8Rng::from_entropy(),
}
}
/// Convert serde_json Value to Ruby Value
fn value_to_ruby_hash(&self, ruby: &Ruby, record: &Map<String, serde_json::Value>) -> Result<Value, Error> {
let hash = ruby.hash_new();
for (key, value) in record {
let ruby_key = Symbol::new(key);
let ruby_value = self.json_to_ruby(ruby, value)?;
hash.aset(ruby_key, ruby_value)?;
}
Ok(hash.as_value())
}
/// Convert serde_json Value to Ruby Value
fn json_to_ruby(&self, ruby: &Ruby, value: &serde_json::Value) -> Result<Value, Error> {
match value {
serde_json::Value::Null => Ok(ruby.qnil().as_value()),
serde_json::Value::Bool(b) => Ok(if *b { ruby.qtrue() } else { ruby.qfalse() }.as_value()),
serde_json::Value::Number(n) => {
if let Some(i) = n.as_i64() {
Ok(ruby.integer_from_i64(i).as_value())
} else if let Some(f) = n.as_f64() {
Ok(ruby.float_from_f64(f).as_value())
} else {
Ok(ruby.qnil().as_value())
}
}
serde_json::Value::String(s) => Ok(ruby.str_new(s).as_value()),
serde_json::Value::Array(arr) => {
let ruby_arr = ruby.ary_new_capa(arr.len());
for item in arr {
ruby_arr.push(self.json_to_ruby(ruby, item)?)?;
}
Ok(ruby_arr.as_value())
}
serde_json::Value::Object(obj) => {
let ruby_hash = ruby.hash_new();
for (k, v) in obj {
ruby_hash.aset(Symbol::new(k), self.json_to_ruby(ruby, v)?)?;
}
Ok(ruby_hash.as_value())
}
}
}
}
4.4 Schema Parser: src/schema.rs
// Optional: Add schema parsing if needed
// For now, the Ruby side handles parsing and passes field specs
5. Building the Extension
5.1 Build Commands
# Install dependencies
bundle install
# Compile the native extension
bundle exec rake compile
# Or build manually
cd ext/fake_data_dsl_native
cargo build --release
5.2 Development Workflow
# Build and test
bundle exec rake compile
bundle exec rspec
# Clean and rebuild
bundle exec rake clobber
bundle exec rake compile
6. Integrating with Ruby
6.1 Create lib/fake_data_dsl/native_engine.rb
# frozen_string_literal: true
module FakeDataDSL
# High-performance native engine using Rust + fake-rs
class NativeEngine
class << self
# Check if native extension is available
def available?
@available ||= begin
require "fake_data_dsl/fake_data_dsl_native"
defined?(FakeDataDSL::Native) && Native.respond_to?(:loaded?) && Native.loaded?
rescue LoadError
false
end
end
# Get available types in native engine
def available_types
return [] unless available?
Native.available_types
end
# Get available locales in native engine
def available_locales
return [] unless available?
Native.available_locales
end
# Get native extension version
def version
return nil unless available?
Native.version
end
end
attr_reader :schema, :locale
# Initialize with a schema
# @param schema [FakeDataDSL::Schema] The schema to generate from
# @param locale [String] Locale for fake data (en, fr_fr, de_de, etc.)
def initialize(schema, locale: "en")
unless self.class.available?
raise LoadError, "Native extension not available. Install Rust and run `bundle exec rake compile`"
end
@schema = schema
@locale = locale
@field_specs = build_field_specs
end
# Generate multiple records
# @param count [Integer] Number of records to generate
# @param seed [Integer, nil] Random seed for reproducibility
# @return [Array<Hash>] Generated records
def generate_many(count, seed: nil)
Native.generate(@field_specs, count, seed, @locale)
end
# Generate a single record
# @param seed [Integer, nil] Random seed for reproducibility
# @return [Hash] Generated record
def generate(seed: nil)
Native.generate_one(@field_specs, seed, @locale)
end
# Generate records directly to file
# @param count [Integer] Number of records to generate
# @param path [String] Output file path
# @param seed [Integer, nil] Random seed for reproducibility
# @param format [String] Output format: jsonl, json, csv
# @return [Integer] Number of records written
def generate_to_file(count, path, seed: nil, format: "jsonl")
Native.generate_to_file(@field_specs, count, path, seed, @locale, format)
end
# Alias for compatibility
alias_method :to_file, :generate_to_file
private
# Build field specs for native engine
# Format: [name, type_name, args_json, optional, nullable]
def build_field_specs
@schema.fields.map do |field|
[
field.name.to_s,
map_type_name(field.type_name),
(field.type_args || {}).to_json,
field.optional?,
field.nullable?
]
end
end
# Map DSL type names to native engine type names
def map_type_name(type_name)
# Most types have the same name
case type_name.to_s
when "full_name" then "name"
when "phone_number" then "phone"
when "street_address" then "street"
when "zip_code" then "postal_code"
else type_name.to_s
end
end
end
end
6.2 Update Main Module lib/fake_data_dsl.rb
# Add to the module
module FakeDataDSL
class << self
# Check if native engine is available
def native_available?
NativeEngine.available?
end
# Create native engine for a schema
def native_engine(schema, locale: "en")
NativeEngine.new(schema, locale: locale)
end
# Generate using native engine (fastest)
def generate_native(schema_or_dsl, count, seed: nil, locale: "en")
schema = schema_or_dsl.is_a?(String) ? parse(schema_or_dsl) : schema_or_dsl
engine = NativeEngine.new(schema, locale: locale)
engine.generate_many(count, seed: seed)
end
# Generate to file using native engine
def generate_native_to_file(schema_or_dsl, count, path, seed: nil, locale: "en", format: "jsonl")
schema = schema_or_dsl.is_a?(String) ? parse(schema_or_dsl) : schema_or_dsl
engine = NativeEngine.new(schema, locale: locale)
engine.generate_to_file(count, path, seed: seed, format: format)
end
end
end
7. Testing
7.1 Create spec/native_engine_spec.rb
# frozen_string_literal: true
require "spec_helper"
RSpec.describe FakeDataDSL::NativeEngine, if: FakeDataDSL::NativeEngine.available? do
let(:schema) do
FakeDataDSL.parse(<<~DSL)
User:
id: uuid
name: name
email: email
age: number(18..80)
active: boolean
DSL
end
describe ".available?" do
it "returns true when native extension is loaded" do
expect(described_class.available?).to be true
end
end
describe ".available_types" do
it "returns list of supported types" do
types = described_class.available_types
expect(types).to include("name", "email", "uuid", "number")
end
end
describe ".available_locales" do
it "returns list of supported locales" do
locales = described_class.available_locales
expect(locales).to include("en", "fr_fr", "de_de")
end
end
describe "#generate" do
subject(:engine) { described_class.new(schema) }
it "generates a single record" do
record = engine.generate(seed: 42)
expect(record).to be_a(Hash)
expect(record[:id]).to match(/^[0-9a-f-]{36}$/)
expect(record[:name]).to be_a(String)
expect(record[:email]).to include("@")
expect(record[:age]).to be_between(18, 80)
expect(record[:active]).to be_in([true, false])
end
it "produces deterministic output with seed" do
record1 = engine.generate(seed: 42)
record2 = engine.generate(seed: 42)
expect(record1).to eq(record2)
end
end
describe "#generate_many" do
subject(:engine) { described_class.new(schema) }
it "generates multiple records" do
records = engine.generate_many(100, seed: 42)
expect(records.size).to eq(100)
expect(records).to all(be_a(Hash))
end
it "generates unique emails" do
records = engine.generate_many(1000, seed: 42)
emails = records.map { |r| r[:email] }
expect(emails.uniq.size).to eq(1000)
end
end
describe "#generate_to_file" do
subject(:engine) { described_class.new(schema) }
let(:tempfile) { Tempfile.new(["test", ".jsonl"]) }
after { tempfile.unlink }
it "generates JSONL file" do
count = engine.generate_to_file(100, tempfile.path, seed: 42, format: "jsonl")
expect(count).to eq(100)
lines = File.readlines(tempfile.path)
expect(lines.size).to eq(100)
first_record = JSON.parse(lines.first)
expect(first_record).to have_key("id")
expect(first_record).to have_key("name")
end
it "generates CSV file" do
count = engine.generate_to_file(100, tempfile.path, seed: 42, format: "csv")
expect(count).to eq(100)
lines = File.readlines(tempfile.path)
expect(lines.first.strip).to eq("id,name,email,age,active")
expect(lines.size).to eq(101) # header + 100 records
end
end
describe "locale support" do
it "generates Japanese names with ja_jp locale" do
japanese_schema = FakeDataDSL.parse("User:\n name: name")
engine = described_class.new(japanese_schema, locale: "ja_jp")
record = engine.generate(seed: 42)
# Japanese names should contain Japanese characters
expect(record[:name]).to match(/[\p{Hiragana}\p{Katakana}\p{Han}]/)
end
it "generates German names with de_de locale" do
german_schema = FakeDataDSL.parse("User:\n name: name")
engine = described_class.new(german_schema, locale: "de_de")
record = engine.generate(seed: 42)
expect(record[:name]).to be_a(String)
end
end
describe "performance", :benchmark do
it "generates 1 million records quickly" do
start_time = Time.now
engine = described_class.new(schema)
tempfile = Tempfile.new(["benchmark", ".jsonl"])
engine.generate_to_file(1_000_000, tempfile.path, seed: 42)
elapsed = Time.now - start_time
records_per_sec = 1_000_000 / elapsed
puts "\nNative engine: #{records_per_sec.round} records/sec"
expect(elapsed).to be < 10 # Should complete in under 10 seconds
tempfile.unlink
end
end
end
8. Packaging the Gem
8.1 Update .gemspec
Gem::Specification.new do |spec|
spec.name = "fake_data_dsl"
spec.version = FakeDataDSL::VERSION
# ... other fields ...
# Mark as having native extensions
spec.extensions = ["ext/fake_data_dsl_native/extconf.rb"]
# Required Ruby version
spec.required_ruby_version = ">= 3.0"
# Dependencies
spec.add_dependency "rb_sys", "~> 0.9"
# Include all necessary files
spec.files = Dir[
"lib/**/*.rb",
"ext/**/*.{rs,toml,rb}",
"*.md",
"LICENSE*"
]
end
8.2 Pre-built Binaries (Optional)
For faster installation, you can provide pre-built binaries:
# In Rakefile
require "rb_sys/extensiontask"
RbSys::ExtensionTask.new("fake_data_dsl_native", GEMSPEC) do |ext|
ext.lib_dir = "lib/fake_data_dsl"
ext.cross_compile = true
ext.cross_platform = %w[
x86_64-linux
x86_64-darwin
arm64-darwin
x86_64-linux-musl
]
end
9. Performance Benchmarks
Expected Performance
| Operation | Ruby Engine | Native Engine | Speedup |
|---|---|---|---|
| 1K records | 50ms | 2ms | 25x |
| 10K records | 500ms | 5ms | 100x |
| 100K records | 5s | 30ms | 166x |
| 1M records | 50s | 300ms | 166x |
| 10M records | 8min | 2s | 240x |
Run Benchmarks
require "benchmark"
require "fake_data_dsl"
schema = FakeDataDSL.parse(<<~DSL)
User:
id: uuid
name: name
email: email
age: number(18..80)
active: boolean
DSL
Benchmark.bm(20) do |x|
x.report("Ruby (1K):") { schema.generate_many(1_000) }
if FakeDataDSL::NativeEngine.available?
engine = FakeDataDSL::NativeEngine.new(schema)
x.report("Native (1K):") { engine.generate_many(1_000) }
x.report("Native (100K):") { engine.generate_many(100_000) }
x.report("Native (1M):") { engine.generate_many(1_000_000) }
end
end
10. Troubleshooting
Common Issues
1. "Rust not found"
# Install Rust
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
source ~/.cargo/env
2. "rb_sys not found"
gem install rb_sys
bundle install
3. Compilation errors
# Clean and rebuild
bundle exec rake clobber
bundle exec rake compile
# Check Rust version
rustc --version # Should be 1.70+
4. "undefined symbol" on Linux
# Install development headers
sudo apt-get install ruby-dev libclang-dev
5. macOS architecture issues
# For Apple Silicon
rustup target add aarch64-apple-darwin
# For Intel
rustup target add x86_64-apple-darwin
Summary
With this integration:
- Install Rust and create the extension structure
- Write Rust code using fake-rs for generation
- Build with
bundle exec rake compile - Use via
FakeDataDSL::NativeEngine
# Check availability
if FakeDataDSL::NativeEngine.available?
engine = FakeDataDSL::NativeEngine.new(schema, locale: "en")
# Generate 10 million records in ~2 seconds
engine.generate_to_file(10_000_000, "data.jsonl")
end
Last updated: January 2026