Module: DataRedactor
- Defined in:
- lib/data_redactor.rb,
lib/data_redactor/version.rb,
lib/data_redactor/integrations/rack.rb,
lib/data_redactor/integrations/rails.rb,
lib/data_redactor/integrations/logger.rb,
ext/data_redactor/data_redactor.c
Overview
High-performance regex-based redactor for sensitive data.
DataRedactor scans text for sensitive patterns (API keys, IBANs, national IDs, emails, phone numbers, etc.) and replaces matches with a configurable placeholder. The matching is done by a C extension backed by POSIX regex.h, so it is fast enough to run inline on large payloads.
Defined Under Namespace
Modules: Integrations Classes: InvalidPatternError, UnknownPatternError, UnknownTagError
Constant Summary collapse
- TAGS =
{ credentials: TAG_CREDENTIALS, financial: TAG_FINANCIAL, tax_id: TAG_TAX_ID, national_id: TAG_NATIONAL_ID, contact: TAG_CONTACT, network: TAG_NETWORK, travel: TAG_TRAVEL, other: TAG_OTHER, custom: TAG_CUSTOM }.freeze
- CAPTURE_GROUP_RE =
This constant is part of a private API. You should avoid using this constant if possible, as it may be removed or be changed in the future.
Capture groups break boundary-wrapper group index assumptions ([1],,[3] shift).
/(?<!\\)\((?!\?:)/.freeze
- RUBY_ONLY_SYNTAX_RE =
This constant is part of a private API. You should avoid using this constant if possible, as it may be removed or be changed in the future.
Ruby regex syntax that has no POSIX ERE equivalent.
/\\[dDwWsShHbB]|\(\?[<!=]|\(\?<[a-zA-Z]|\(\?[imx]|[*+?]\?/.freeze
- PLACEHOLDER_DEFAULT =
Default placeholder used when
placeholder:is not given to redact. "[REDACTED]"- VERSION =
Current gem version. Follows Semantic Versioning 2.0.0.
"0.7.1"- BUILTIN_PATTERN_NAMES =
rb_ary_freeze(builtin_names)
- BUILTIN_PATTERN_TAG_BITS =
rb_ary_freeze(builtin_tag_bits)
- PH_MODE_PLAIN =
Placeholder mode constants.
INT2NUM(PLACEHOLDER_MODE_PLAIN)
- PH_MODE_TAGGED =
INT2NUM(PLACEHOLDER_MODE_TAGGED)
- PH_MODE_HASH =
INT2NUM(PLACEHOLDER_MODE_HASH)
- TAG_CREDENTIALS =
Tag bitmask values used by the Ruby wrapper to build only/except masks.
INT2NUM(TAG_CREDENTIALS)
- TAG_FINANCIAL =
INT2NUM(TAG_FINANCIAL)
- TAG_TAX_ID =
INT2NUM(TAG_TAX_ID)
- TAG_NATIONAL_ID =
INT2NUM(TAG_NATIONAL_ID)
- TAG_CONTACT =
INT2NUM(TAG_CONTACT)
- TAG_NETWORK =
INT2NUM(TAG_NETWORK)
- TAG_TRAVEL =
INT2NUM(TAG_TRAVEL)
- TAG_OTHER =
INT2NUM(TAG_OTHER)
- TAG_CUSTOM =
INT2NUM(TAG_CUSTOM)
- TAG_ALL =
INT2NUM(TAG_ALL)
Class Method Summary collapse
-
._add_pattern ⇒ Object
Note: _redact(text, ph_mode, ph_str, enable_bits) and _scan(text, enable_bits).
- ._clear_custom_patterns ⇒ Object
- ._custom_patterns ⇒ Object
- ._redact(rb_text, rb_ph_mode, rb_ph_str, rb_enable_bits) ⇒ Object
- ._remove_pattern ⇒ Object
- ._scan(rb_text, rb_enable_bits) ⇒ Object
-
.add_pattern(name:, regex:, tag: :custom, boundary: false) ⇒ Boolean
Register a custom redaction pattern.
-
.build_enable_bits(only, except) ⇒ Array<Integer>
private
Build the per-pattern enable bit-list passed to the C layer.
-
.clear_custom_patterns! ⇒ nil
Remove every registered custom pattern.
-
.custom_patterns ⇒ Array<Hash{Symbol => Object}>
List every currently registered custom pattern.
- .pattern_enabled?(name, tag_bit, only_present, only_bits, only_names, except_bits, except_names) ⇒ Boolean private
-
.pattern_names ⇒ Array<String>
List of every pattern name the redactor knows about.
-
.redact(text, only: nil, except: nil, placeholder: PLACEHOLDER_DEFAULT) ⇒ String
Redact every match of the configured patterns in
text. -
.remove_pattern(name) ⇒ Boolean
Remove a previously registered custom pattern.
-
.resolve_placeholder(placeholder) ⇒ Array(Integer, String)
private
Translate the user-facing
placeholder:value into the (mode_int, str) pair the C layer expects. -
.scan(text, only: nil, except: nil) ⇒ Hash{Symbol => Object}
Scan
textand return both the redacted string and per-match metadata. -
.split_filter(entries) ⇒ Array(Integer, Set<String>)
private
Split a mixed Symbol/String filter list into (tag_bitmask, name_set).
-
.tags ⇒ Array<Symbol>
List of supported tag symbols.
Class Method Details
._add_pattern ⇒ Object
Note: _redact(text, ph_mode, ph_str, enable_bits) and _scan(text, enable_bits).
._clear_custom_patterns ⇒ Object
._custom_patterns ⇒ Object
._redact(rb_text, rb_ph_mode, rb_ph_str, rb_enable_bits) ⇒ Object
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 |
# File 'ext/data_redactor/redact.c', line 116
VALUE rb_data_redactor_redact(VALUE self, VALUE rb_text,
VALUE rb_ph_mode, VALUE rb_ph_str,
VALUE rb_enable_bits) {
Check_Type(rb_text, T_STRING);
Check_Type(rb_ph_str, T_STRING);
Check_Type(rb_enable_bits, T_ARRAY);
int ph_mode = NUM2INT(rb_ph_mode);
const char *ph_str_plain = StringValueCStr(rb_ph_str);
const char *input = StringValueCStr(rb_text);
char *working = strdup(input);
if (!working) rb_raise(rb_eNoMemError, "strdup failed");
placeholder_t ph;
ph.mode = ph_mode;
for (int i = 0; i < NUM_PATTERNS; i++) {
if (!enable_bit(rb_enable_bits, i)) continue;
ph.str = (ph_mode == PLACEHOLDER_MODE_PLAIN)
? ph_str_plain
: tag_name_for_bit(pattern_tags[i]);
char *result = replace_all_matches(&compiled_patterns[i], working,
boundary_wrapped[i], &ph);
free(working);
if (!result) rb_raise(rb_eNoMemError, "replace_all_matches allocation failed");
working = result;
}
for (int i = 0; i < custom_count; i++) {
if (!enable_bit(rb_enable_bits, NUM_PATTERNS + i)) continue;
ph.str = (ph_mode == PLACEHOLDER_MODE_PLAIN)
? ph_str_plain
: tag_name_for_bit(custom_patterns[i].tag);
char *result = replace_all_matches(&custom_patterns[i].compiled, working,
custom_patterns[i].boundary, &ph);
free(working);
if (!result) rb_raise(rb_eNoMemError, "replace_all_matches allocation failed (custom)");
working = result;
}
VALUE rb_result = rb_str_new_cstr(working);
free(working);
return rb_result;
}
|
._remove_pattern ⇒ Object
._scan(rb_text, rb_enable_bits) ⇒ Object
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 |
# File 'ext/data_redactor/scan.c', line 29
VALUE rb_data_redactor_scan(VALUE self, VALUE rb_text, VALUE rb_enable_bits) {
Check_Type(rb_text, T_STRING);
Check_Type(rb_enable_bits, T_ARRAY);
const char *input = StringValueCStr(rb_text);
static const placeholder_t ph_default = { PLACEHOLDER_MODE_PLAIN, "[REDACTED]" };
char *working = strdup(input);
if (!working) rb_raise(rb_eNoMemError, "strdup failed");
VALUE matches_arr = rb_ary_new();
typedef struct { long wpos; long orig_len; } repl_t;
repl_t *repl_log = NULL;
int repl_count = 0;
int repl_cap = 0;
#define REPL_LOG_PUSH(_wpos, _olen) do { \
if (repl_count >= repl_cap) { \
int _nc = repl_cap == 0 ? 16 : repl_cap * 2; \
repl_t *_t = (repl_t *)realloc(repl_log, sizeof(repl_t) * _nc); \
if (!_t) { free(repl_log); free(working); rb_raise(rb_eNoMemError, "repl_log"); } \
repl_log = _t; repl_cap = _nc; \
} \
repl_log[repl_count].wpos = (_wpos); \
repl_log[repl_count].orig_len = (_olen); \
repl_count++; \
} while (0)
#define WORKING_TO_ORIG(_wpos) ({ \
long _shift = 0; \
for (int _ri = 0; _ri < repl_count; _ri++) { \
if (repl_log[_ri].wpos <= (_wpos)) \
_shift += 10 - repl_log[_ri].orig_len; \
} \
(_wpos) - _shift; \
})
#define COLLECT_AND_REPLACE(pat, use_bnd, tag_bit, pat_name) do { \
const char *_cur = working; \
regmatch_t _m[4]; \
while (regexec((pat), _cur, 4, _m, 0) == 0) { \
regoff_t _fso = _m[0].rm_so, _feo = _m[0].rm_eo; \
if (_fso < 0 || _feo < _fso) break; \
regoff_t _cso = _fso, _ceo = _feo; \
if (use_bnd) { \
if (_m[1].rm_so >= 0 && _m[1].rm_eo > _m[1].rm_so) \
_cso = _m[1].rm_eo; \
if (_m[3].rm_so >= 0 && _m[3].rm_eo > _m[3].rm_so) \
_ceo = _m[3].rm_so; \
} \
size_t _vlen = (size_t)(_ceo - _cso); \
long _wpos = (long)(_cur - working) + (long)_cso; \
long _orig = WORKING_TO_ORIG(_wpos); \
VALUE _match = rb_hash_new(); \
rb_hash_aset(_match, ID2SYM(rb_intern("tag")), \
ID2SYM(rb_intern(tag_name_for_bit(tag_bit)))); \
rb_hash_aset(_match, ID2SYM(rb_intern("name")), \
rb_str_new_cstr(pat_name)); \
rb_hash_aset(_match, ID2SYM(rb_intern("value")), \
rb_str_new(_cur + _cso, _vlen)); \
rb_hash_aset(_match, ID2SYM(rb_intern("start")), \
LONG2NUM(_orig)); \
rb_hash_aset(_match, ID2SYM(rb_intern("length")), \
LONG2NUM((long)_vlen)); \
rb_ary_push(matches_arr, _match); \
REPL_LOG_PUSH(_wpos, (long)_vlen); \
if (_feo == _fso) { if (*_cur) _cur++; else break; } \
else _cur += _feo; \
} \
char *_next = replace_all_matches((pat), working, (use_bnd), &ph_default); \
free(working); \
if (!_next) { free(repl_log); rb_raise(rb_eNoMemError, "replace_all_matches failed in scan"); } \
working = _next; \
} while (0)
for (int i = 0; i < NUM_PATTERNS; i++) {
if (!scan_enable_bit(rb_enable_bits, i)) continue;
COLLECT_AND_REPLACE(&compiled_patterns[i], boundary_wrapped[i],
pattern_tags[i], pattern_names[i]);
}
for (int i = 0; i < custom_count; i++) {
if (!scan_enable_bit(rb_enable_bits, NUM_PATTERNS + i)) continue;
COLLECT_AND_REPLACE(&custom_patterns[i].compiled,
custom_patterns[i].boundary,
custom_patterns[i].tag, custom_patterns[i].name);
}
#undef COLLECT_AND_REPLACE
#undef WORKING_TO_ORIG
#undef REPL_LOG_PUSH
free(repl_log);
VALUE result = rb_hash_new();
VALUE rb_redacted = rb_str_new_cstr(working);
free(working);
rb_hash_aset(result, ID2SYM(rb_intern("redacted")), rb_redacted);
rb_hash_aset(result, ID2SYM(rb_intern("matches")), matches_arr);
return result;
}
|
.add_pattern(name:, regex:, tag: :custom, boundary: false) ⇒ Boolean
Register a custom redaction pattern.
Patterns must be valid POSIX ERE. Ruby-only syntax (\d, \s, \w, \b, lookaround, non-greedy quantifiers, named groups) is rejected at registration time, never at redaction time.
If a pattern with the same name is already registered, it is replaced (the old compiled regex_t is freed).
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 |
# File 'lib/data_redactor.rb', line 194 def add_pattern(name:, regex:, tag: :custom, boundary: false) raise ArgumentError, "name must be a non-empty String" \ unless name.is_a?(String) && !name.empty? source = case regex when String then regex when Regexp then regex.source else raise ArgumentError, "regex must be a String or Regexp, got #{regex.class}" end if source =~ RUBY_ONLY_SYNTAX_RE raise InvalidPatternError, "pattern #{name.inspect} uses Ruby-only syntax (#{$&.inspect}); " \ "use POSIX ERE — no \\d, \\s, \\w, \\b, lookaround, non-greedy, or named groups" end if boundary && source =~ CAPTURE_GROUP_RE raise InvalidPatternError, "pattern #{name.inspect} has capture groups and cannot use boundary: true" end tag_bit = TAGS[tag] or raise UnknownTagError, "unknown tag #{tag.inspect}; valid tags: #{TAGS.keys.inspect}" _add_pattern(name, source, tag_bit, boundary ? 1 : 0) end |
.build_enable_bits(only, except) ⇒ Array<Integer>
This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.
Build the per-pattern enable bit-list passed to the C layer.
The list has one Integer (0 or 1) per pattern in execution order: built-ins first (NUM_PATTERNS entries), then currently registered custom patterns in registration order. C iterates by index and skips zeros.
Semantics of only: / except: — both accept a mix of Symbols (tags) and Strings (pattern names):
enabled(p) iff
(only is nil OR p.tag ∈ only_tags OR p.name ∈ only_names)
AND p.tag ∉ except_tags AND p.name ∉ except_names
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 |
# File 'lib/data_redactor.rb', line 297 def build_enable_bits(only, except) only_bits, only_names = split_filter(only) except_bits, except_names = split_filter(except) only_present = !only.nil? bits = Array.new(BUILTIN_PATTERN_NAMES.length + _custom_patterns.length, 0) BUILTIN_PATTERN_NAMES.each_with_index do |name, i| tag_bit = BUILTIN_PATTERN_TAG_BITS[i] bits[i] = 1 if pattern_enabled?(name, tag_bit, only_present, only_bits, only_names, except_bits, except_names) end _custom_patterns.each_with_index do |h, i| bits[BUILTIN_PATTERN_NAMES.length + i] = 1 if pattern_enabled?( h[:name], h[:tag_bit], only_present, only_bits, only_names, except_bits, except_names) end bits end |
.clear_custom_patterns! ⇒ nil
Remove every registered custom pattern.
Mostly useful in test suites that need a clean slate between examples.
247 248 249 |
# File 'lib/data_redactor.rb', line 247 def clear_custom_patterns! _clear_custom_patterns end |
.custom_patterns ⇒ Array<Hash{Symbol => Object}>
List every currently registered custom pattern.
235 236 237 238 239 240 |
# File 'lib/data_redactor.rb', line 235 def custom_patterns _custom_patterns.map do |h| { name: h[:name], source: h[:source], tag: TAGS.key(h[:tag_bit]) || :custom, boundary: h[:boundary] } end end |
.pattern_enabled?(name, tag_bit, only_present, only_bits, only_names, except_bits, except_names) ⇒ Boolean
This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.
321 322 323 324 325 326 327 328 |
# File 'lib/data_redactor.rb', line 321 def pattern_enabled?(name, tag_bit, only_present, only_bits, only_names, except_bits, except_names) return false if (tag_bit & except_bits) != 0 return false if except_names.include?(name) return true unless only_present return true if (tag_bit & only_bits) != 0 only_names.include?(name) end |
.pattern_names ⇒ Array<String>
List of every pattern name the redactor knows about.
Includes the BUILTIN_PATTERN_NAMES plus any names registered via add_pattern. Useful for discovering what String values only: / except: accept, and for filtering / debugging.
92 93 94 |
# File 'lib/data_redactor.rb', line 92 def pattern_names BUILTIN_PATTERN_NAMES + _custom_patterns.map { |h| h[:name] } end |
.redact(text, only: nil, except: nil, placeholder: PLACEHOLDER_DEFAULT) ⇒ String
Redact every match of the configured patterns in text.
only: and except: both accept a single value or an Array, mixing:
-
Symbols — tag names from TAGS (e.g.
:contact,:credentials). -
Strings — specific pattern names from pattern_names (e.g. “email”).
They can be combined: only: :contact, except: [“email”] means “redact every contact pattern except email.” Symbols give you tag-level control; Strings give you per-pattern precision.
Precedence: a pattern is redacted iff (only is nil OR pattern matches only:) AND (pattern does not match except:). except: always wins over only: when they overlap — e.g. only: :contact, except: :contact produces an empty redaction (no-op), and only: [“email”], except: [“email”] likewise skips email entirely.
130 131 132 133 134 |
# File 'lib/data_redactor.rb', line 130 def redact(text, only: nil, except: nil, placeholder: PLACEHOLDER_DEFAULT) enable_bits = build_enable_bits(only, except) ph_mode, ph_str = resolve_placeholder(placeholder) _redact(text, ph_mode, ph_str, enable_bits) end |
.remove_pattern(name) ⇒ Boolean
Remove a previously registered custom pattern.
226 227 228 |
# File 'lib/data_redactor.rb', line 226 def remove_pattern(name) _remove_pattern(name.to_s) end |
.resolve_placeholder(placeholder) ⇒ Array(Integer, String)
This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.
Translate the user-facing placeholder: value into the (mode_int, str) pair the C layer expects.
337 338 339 340 341 342 343 344 345 346 |
# File 'lib/data_redactor.rb', line 337 def resolve_placeholder(placeholder) case placeholder when :tagged then [PH_MODE_TAGGED, ""] when :hash then [PH_MODE_HASH, ""] when String then [PH_MODE_PLAIN, placeholder] else raise ArgumentError, "placeholder must be a String, :tagged, or :hash — got #{placeholder.inspect}" end end |
.scan(text, only: nil, except: nil) ⇒ Hash{Symbol => Object}
Scan text and return both the redacted string and per-match metadata.
Useful for auditing, false-positive tuning, and compliance pipelines. :start and :length are byte offsets into the original string, so text.byteslice(m, m) == m.
156 157 158 159 160 161 162 |
# File 'lib/data_redactor.rb', line 156 def scan(text, only: nil, except: nil) enable_bits = build_enable_bits(only, except) result = _scan(text, enable_bits) # Normalise: convert tag string from C (uppercase) back to the Symbol used in TAGS result[:matches].each { |m| m[:tag] = m[:tag].to_s.downcase.to_sym } result end |
.split_filter(entries) ⇒ Array(Integer, Set<String>)
This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.
Split a mixed Symbol/String filter list into (tag_bitmask, name_set).
259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 |
# File 'lib/data_redactor.rb', line 259 def split_filter(entries) bits = 0 names = Set.new return [bits, names] if entries.nil? Array(entries).each do |e| case e when Symbol bit = TAGS[e] or raise UnknownTagError, "unknown tag #{e.inspect}; valid tags: #{TAGS.keys.inspect}" bits |= bit when String unless pattern_names.include?(e) raise UnknownPatternError, "unknown pattern name #{e.inspect}; see DataRedactor.pattern_names" end names << e else raise ArgumentError, "only:/except: entries must be a Symbol (tag) or String (pattern name), got #{e.inspect}" end end [bits, names] end |
.tags ⇒ Array<Symbol>
List of supported tag symbols.
80 81 82 |
# File 'lib/data_redactor.rb', line 80 def TAGS.keys end |