Module: Gsplat

Defined in:
lib/gsplat.rb,
lib/gsplat/utils.rb,
lib/gsplat/io/npy.rb,
lib/gsplat/io/ply.rb,
lib/gsplat/native.rb,
lib/gsplat/backend.rb,
lib/gsplat/version.rb,
lib/gsplat/io/image.rb,
lib/gsplat/math/mat.rb,
lib/gsplat/io/colmap.rb,
lib/gsplat/math/ssim.rb,
lib/gsplat/native_ops.rb,
lib/gsplat/optim/adam.rb,
lib/gsplat/strategy/ops.rb,
lib/gsplat/io/checkpoint.rb,
lib/gsplat/io/ply_reader.rb,
lib/gsplat/rasterization.rb,
lib/gsplat/strategy/base.rb,
lib/gsplat/strategy/mcmc.rb,
lib/gsplat/io/colmap_text.rb,
lib/gsplat/io/zip_archive.rb,
lib/gsplat/ops/accumulate.rb,
lib/gsplat/ops/projection.rb,
lib/gsplat/ops/relocation.rb,
lib/gsplat/training/scene.rb,
lib/gsplat/compression/png.rb,
lib/gsplat/math/quaternion.rb,
lib/gsplat/ops/isect_tiles.rb,
lib/gsplat/training/config.rb,
lib/gsplat/training/losses.rb,
lib/gsplat/autograd/context.rb,
lib/gsplat/io/colmap_binary.rb,
lib/gsplat/strategy/default.rb,
lib/gsplat/training/trainer.rb,
lib/gsplat/autograd/function.rb,
lib/gsplat/autograd/variable.rb,
lib/gsplat/io/image_backends.rb,
lib/gsplat/native_raster_ops.rb,
lib/gsplat/strategy/mcmc_ops.rb,
lib/gsplat/compression/kmeans.rb,
lib/gsplat/optim/lr_scheduler.rb,
lib/gsplat/rasterization_2dgs.rb,
lib/gsplat/ops/eval3d_rasterize.rb,
lib/gsplat/ops/tensor_shape_ops.rb,
lib/gsplat/ops/tensor_value_ops.rb,
lib/gsplat/optim/selective_adam.rb,
lib/gsplat/compression/grid_sort.rb,
lib/gsplat/compression/png_codec.rb,
lib/gsplat/compression/quantizer.rb,
lib/gsplat/rasterization_helpers.rb,
lib/gsplat/training/image_fitter.rb,
lib/gsplat/math/camera_distortion.rb,
lib/gsplat/math/camera_projection.rb,
lib/gsplat/backend/ruby/accumulate.rb,
lib/gsplat/backend/ruby/projection.rb,
lib/gsplat/ops/rasterize_to_pixels.rb,
lib/gsplat/ops/spherical_harmonics.rb,
lib/gsplat/backend/ruby/isect_tiles.rb,
lib/gsplat/rasterization_validation.rb,
lib/gsplat/backend/ruby/tile_compositor.rb,
lib/gsplat/math/small_matrix_primitives.rb,
lib/gsplat/math/spherical_harmonic_basis.rb,
lib/gsplat/ops/quat_scale_to_covar_preci.rb,
lib/gsplat/backend/ruby/eval3d_rasterizer.rb,
lib/gsplat/backend/ruby/accumulate_backward.rb,
lib/gsplat/backend/ruby/projection_backward.rb,
lib/gsplat/backend/ruby/rasterize_to_pixels.rb,
lib/gsplat/backend/ruby/spherical_harmonics.rb,
lib/gsplat/backend/ruby/projection_input_vjp.rb,
lib/gsplat/ops/rasterize_to_indices_in_range.rb,
lib/gsplat/backend/ruby/tile_compositor_backward.rb,
lib/gsplat/backend/ruby/projection_covariance_vjp.rb,
lib/gsplat/backend/ruby/quat_scale_to_covar_preci.rb,
lib/gsplat/backend/ruby/rasterize_to_pixels_backward.rb,
lib/gsplat/backend/ruby/rasterize_to_indices_in_range.rb

Overview

Differentiable Gaussian rasterization primitives.

Defined Under Namespace

Modules: Autograd, Backend, Compression, IO, Math, Native, NativeOps, NativeRasterOps, Ops, Optim, Rasterization, Rasterization2DGS, RasterizationHelpers, RasterizationValidation, Strategy, Training, Utils Classes: Error, NotSupportedError, ShapeError

Constant Summary collapse

VERSION =

Semantic package version.

"1.0.0"

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.logger ⇒ Logger

Logger used for backend fallbacks and unsupported option warnings.

Returns:

  • (Logger) 


72
73
74
 
# File 'lib/gsplat.rb', line 72

def logger
  @logger ||= Logger.new($stderr, level: Logger::WARN)
end

.rng ⇒ Random

Shared deterministic random source used by initialization and strategies.

Returns:

  • (Random) 


79
80
81
 
# File 'lib/gsplat.rb', line 79

def rng
  @rng ||= Random.new
end

Class Method Details

.accumulate(means2d, conics, opacities, colors, width:, height:, backgrounds: nil) ⇒ Object

Composites every Gaussian over every pixel without tile acceleration. rubocop:disable Metrics/ParameterLists



52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/gsplat/ops/accumulate.rb', line 52

def accumulate(means2d, conics, opacities, colors, width:, height:, backgrounds: nil)
  # rubocop:enable Metrics/ParameterLists
  inputs = [means2d, conics, opacities, colors, backgrounds]
  return Ops::Accumulate.apply(*inputs, width, height) if inputs.any?(Autograd::Variable)

  Backend.dispatch(
    :accumulate_forward,
    *inputs,
    width,
    height
  ).first(2)
end

.backend ⇒ Symbol

Active operation backend.

Returns:

  • (Symbol) —

    :auto, :ruby, or :native



86
87
88
 
# File 'lib/gsplat.rb', line 86

def backend
  @backend ||= Backend.normalize_backend(ENV.fetch("GSPLAT_BACKEND", "auto"))
end

.backend=(value) ⇒ Symbol

Selects the operation backend.

Parameters:

  • value (Symbol, String) —

    :auto, :ruby, or :native

Returns:

  • (Symbol) —

    normalized backend



94
95
96
 
# File 'lib/gsplat.rb', line 94

def backend=(value)
  @backend = Backend.normalize_backend(value)
end

.fully_fused_projection(means, viewmats:, ks:, width:, height:, covars: nil, quats: nil, scales: nil, eps2d: 0.3, near_plane: 0.01, far_plane: 1e10, radius_clip: 0.0, calc_compensations: false, camera_model: "pinhole", radial_coeffs: nil, tangential_coeffs: nil, thin_prism_coeffs: nil, global_z_order: true) ⇒ Array<(Numo::NArray, Autograd::Variable, nil)>

Projects and culls a dense camera batch.

Inputs use float32/float64 Numo arrays or Gsplat::Autograd::Variable; geometry is [N,3], views [C,4,4], intrinsics [C,3,3], and outputs are radii [C,N,2], means [C,N,2], depths [C,N], conics [C,N,3], plus optional compensations [C,N].

rubocop:disable Metrics/ParameterLists, Naming/MethodParameterName

Returns:



106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/gsplat/ops/projection.rb', line 106

def fully_fused_projection(means, viewmats:, ks:, width:, height:, covars: nil, quats: nil, scales: nil,
                           eps2d: 0.3, near_plane: 0.01, far_plane: 1e10, radius_clip: 0.0,
                           calc_compensations: false, camera_model: "pinhole",
                           radial_coeffs: nil, tangential_coeffs: nil, thin_prism_coeffs: nil,
                           global_z_order: true)
  # rubocop:enable Metrics/ParameterLists, Naming/MethodParameterName
  inputs = [means, covars, quats, scales, viewmats, ks]
  options = {
    eps2d: eps2d,
    near_plane: near_plane,
    far_plane: far_plane,
    radius_clip: radius_clip,
    calc_compensations: calc_compensations,
    camera_model: camera_model,
    radial_coeffs: radial_coeffs,
    tangential_coeffs: tangential_coeffs,
    thin_prism_coeffs: thin_prism_coeffs,
    global_z_order: global_z_order
  }
  if inputs.any?(Autograd::Variable)
    return Ops::FullyFusedProjection.apply(
      *inputs,
      width,
      height,
      **options
    )
  end

  Backend.dispatch(
    :fully_fused_projection_forward,
    *inputs,
    width,
    height,
    **options
  )
end

.fully_fused_projection_2dgs(means, quats:, scales:, viewmats:, ks:, width:, height:, eps2d: 0.3, near_plane: 0.01, far_plane: 1e10, radius_clip: 0.0) ⇒ Object

Projection metadata used by the 2DGS rasterizer. rubocop:disable Metrics/ParameterLists, Naming/MethodParameterName



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/gsplat/rasterization_2dgs.rb', line 121

def fully_fused_projection_2dgs(means, quats:, scales:, viewmats:, ks:, width:, height:,
                                eps2d: 0.3, near_plane: 0.01, far_plane: 1e10,
                                radius_clip: 0.0)
  # rubocop:enable Metrics/ParameterLists, Naming/MethodParameterName
  radii, means2d, depths, conics, = fully_fused_projection(
    means, quats: quats, scales: scales, viewmats: viewmats, ks: ks, width: width,
           height: height, eps2d: eps2d, near_plane: near_plane, far_plane: far_plane,
           radius_clip: radius_clip
  )
  normals = Rasterization2DGS.send(
    :camera_normals, Ops::TensorOps.data(means), Ops::TensorOps.data(quats),
    Ops::TensorOps.data(viewmats)
  )
  ray_transforms = Ops::TensorOps.data(conics).class.zeros(
    *(Ops::TensorOps.data(conics).shape[0...-1] + [3, 3])
  )
  [radii, means2d, depths, ray_transforms, normals]
end

.isect_offset_encode(isect_ids, camera_count, tile_width, tile_height) ⇒ Object

Encodes the starting intersection index for every camera tile.



25
26
27
28
29
30
31
32
33
 
# File 'lib/gsplat/ops/isect_tiles.rb', line 25

def isect_offset_encode(isect_ids, camera_count, tile_width, tile_height)
  Backend.dispatch(
    :isect_offset_encode,
    isect_ids,
    camera_count,
    tile_width,
    tile_height
  )
end

.isect_tiles(means2d, radii, depths, tile_size, tile_width, tile_height, sort: true) ⇒ Object

Enumerates Gaussian/tile intersections. rubocop:disable Metrics/ParameterLists



10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/gsplat/ops/isect_tiles.rb', line 10

def isect_tiles(means2d, radii, depths, tile_size, tile_width, tile_height, sort: true)
  # rubocop:enable Metrics/ParameterLists
  Backend.dispatch(
    :isect_tiles,
    means2d,
    radii,
    depths,
    tile_size,
    tile_width,
    tile_height,
    sort: sort
  )
end

.ortho_proj(means, covars, intrinsics, width, height) ⇒ Array<Numo::NArray>

Projects camera-space means/covariances with an orthographic model.

Parameters:

  • means (Numo::NArray) —

    [C,N,3]

  • covars (Numo::NArray) —

    [C,N,3,3]

  • intrinsics (Numo::NArray) —

    [C,3,3]

Returns:

  • (Array<Numo::NArray>) —

    projected means [C,N,2] and covariances [C,N,2,2]



93
94
95
 
# File 'lib/gsplat/ops/projection.rb', line 93

def ortho_proj(means, covars, intrinsics, width, height)
  Math::CameraProjection.ortho_proj(means, covars, intrinsics, width, height)
end

.persp_proj(means, covars, intrinsics, width, height) ⇒ Array<Numo::NArray>

Projects camera-space means/covariances with a pinhole model.

Parameters:

  • means (Numo::NArray) —

    [C,N,3]

  • covars (Numo::NArray) —

    [C,N,3,3]

  • intrinsics (Numo::NArray) —

    [C,3,3]

Returns:

  • (Array<Numo::NArray>) —

    projected means [C,N,2] and covariances [C,N,2,2]



83
84
85
 
# File 'lib/gsplat/ops/projection.rb', line 83

def persp_proj(means, covars, intrinsics, width, height)
  Math::CameraProjection.persp_proj(means, covars, intrinsics, width, height)
end

.quat_scale_to_covar_preci(quaternions, scales, compute_covar: true, compute_preci: true, triu: false) ⇒ Array<(Autograd::Variable, Numo::NArray, nil)>

Converts wxyz quaternions and scales to covariance and/or precision matrices.

Parameters:

  • quaternions (Autograd::Variable, Numo::NArray) —

    [...,4]

  • scales (Autograd::Variable, Numo::NArray) —

    [...,3]

  • compute_covar (Boolean) (defaults to: true)
  • compute_preci (Boolean) (defaults to: true)
  • triu (Boolean) (defaults to: false) —

    return [...,6] upper triangles when true

Returns:

Raises:

  • (ArgumentError) 


84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/gsplat/ops/quat_scale_to_covar_preci.rb', line 84

def quat_scale_to_covar_preci(quaternions, scales, compute_covar: true, compute_preci: true, triu: false)
  if [quaternions, scales].any?(Autograd::Variable)
    return Ops::QuatScaleToCovarPreci.apply(
      quaternions,
      scales,
      compute_covar: compute_covar,
      compute_preci: compute_preci,
      triu: triu
    )
  end

  raise ArgumentError, "at least one output must be requested" unless compute_covar || compute_preci

  Backend.dispatch(
    :quat_scale_to_covar_preci_forward,
    quaternions,
    scales,
    compute_covar: compute_covar,
    compute_preci: compute_preci,
    triu: triu
  )
end

.rasterization ⇒ Array

Renders a dense batch of cameras with differentiable 3D Gaussians.

Geometry uses float32/float64 arrays means [N,3], quats [N,4], scales [N,3]; colors are [N,D] or SH [N,K,D], views are [C,4,4], and intrinsics are [C,3,3]. Activated opacities are [N].

Returns:

  • (Array) —

    rendered colors [C,H,W,X], alphas [C,H,W,1], and metadata Hash



126
127
128
 
# File 'lib/gsplat/rasterization.rb', line 126

def rasterization(**)
  Rasterization.render(**)
end

.rasterization_2dgs ⇒ Array

Rasterizes oriented 2D Gaussian surfels.

Returns:

  • (Array) —

    colors, alphas, normals, surface normals, distortion, median depth, metadata



115
116
117
 
# File 'lib/gsplat/rasterization_2dgs.rb', line 115

def rasterization_2dgs(**)
  Rasterization2DGS.render(**)
end

.rasterize_to_indices_in_range(range_start, range_end, transmittances, means2d, conics, opacities, width, height, tile_size, isect_offsets, flatten_ids) ⇒ Array<Numo::Int64>

Enumerates Gaussian contributions for tile-list batches in depth order.

rubocop:disable Metrics/ParameterLists

Returns:

  • (Array<Numo::Int64>) —

    Gaussian, pixel, and image IDs



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/gsplat/ops/rasterize_to_indices_in_range.rb', line 14

def rasterize_to_indices_in_range(range_start, range_end, transmittances, means2d, conics, opacities,
                                  width, height, tile_size, isect_offsets, flatten_ids)
  # rubocop:enable Metrics/ParameterLists
  tensors = [transmittances, means2d, conics, opacities, isect_offsets, flatten_ids].map do |value|
    Ops::TensorOps.data(value)
  end
  Backend.dispatch(
    :rasterize_to_indices_in_range,
    range_start,
    range_end,
    *tensors.first(4),
    width,
    height,
    tile_size,
    *tensors.last(2)
  )
end

.rasterize_to_pixels(means2d, conics, colors, opacities, width, height, tile_size, isect_offsets, flatten_ids, backgrounds: nil, masks: nil, absgrad: false) ⇒ Array<(Numo::NArray, Autograd::Variable)>

Alpha-composites sorted tile intersections.

Projected inputs have shapes means2d [C,N,2], conics [C,N,3], colors [C,N,D], and opacities [C,N]. Outputs are color [C,H,W,D] and alpha [C,H,W,1].

rubocop:disable Metrics/ParameterLists

Returns:



82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/gsplat/ops/rasterize_to_pixels.rb', line 82

def rasterize_to_pixels(means2d, conics, colors, opacities, width, height, tile_size,
                        isect_offsets, flatten_ids, backgrounds: nil, masks: nil, absgrad: false)
  # rubocop:enable Metrics/ParameterLists
  inputs = [
    means2d, conics, colors, opacities, backgrounds, masks, width, height,
    tile_size, isect_offsets, flatten_ids
  ]
  return Ops::RasterizeToPixels.apply(*inputs, absgrad: absgrad) if inputs.any?(Autograd::Variable)

  Backend.dispatch(:rasterize_to_pixels_forward, *inputs).first(2)
end

.relocation(opacities, scales, ratios, binoms: nil) ⇒ Object

Computes the deterministic 3DGS-MCMC relocation update.



89
90
91
92
 
# File 'lib/gsplat/ops/relocation.rb', line 89

def relocation(opacities, scales, ratios, binoms: nil)
  options = binoms ? { binoms: binoms } : {}
  Ops::Relocation.compute(opacities, scales, ratios, **options)
end

.spherical_harmonics(degree, directions, coefficients, masks: nil) ⇒ Autograd::Variable, Numo::NArray

Evaluates real spherical harmonics.

Parameters:

  • degree (Integer) —

    degree 0..4

  • directions (Autograd::Variable, Numo::NArray) —

    [...,3]

  • coefficients (Autograd::Variable, Numo::NArray) —

    [...,K,D], K >= (degree+1)^2

  • masks (Numo::Bit, nil) (defaults to: nil) —

    [...]

Returns:



52
53
54
55
56
57
58
 
# File 'lib/gsplat/ops/spherical_harmonics.rb', line 52

def spherical_harmonics(degree, directions, coefficients, masks: nil)
  if [directions, coefficients].any?(Autograd::Variable)
    return Ops::SphericalHarmonics.apply(degree, directions, coefficients, masks: masks)
  end

  Backend.dispatch(:spherical_harmonics_forward, degree, directions, coefficients, masks: masks)
end

.world_to_cam(means, covars, viewmats) ⇒ Array<Numo::NArray>

Transforms world-space means/covariances into camera space.

Parameters:

  • means (Numo::NArray) —

    [N,3]

  • covars (Numo::NArray) —

    [N,3,3] or packed [N,6]

  • viewmats (Numo::NArray) —

    [C,4,4] world-to-camera transforms

Returns:

  • (Array<Numo::NArray>) —

    camera means [C,N,3] and covariances [C,N,3,3]



73
74
75
 
# File 'lib/gsplat/ops/projection.rb', line 73

def world_to_cam(means, covars, viewmats)
  Math::CameraProjection.world_to_cam(means, covars, viewmats)
end