Module: MultiCompress

Defined in:

lib/multi_compress.rb,
lib/multi_compress/codec.rb,
lib/multi_compress/version.rb,
lib/multi_compress/database.rb,
lib/multi_compress/active_record.rb,
lib/multi_compress/db_deployment.rb,
ext/multi_compress/multi_compress.c

Defined Under Namespace

Modules: ActiveRecordSupport, Brotli, DBDeployment, Database, InflaterDefaults, LZ4, Zstd Classes: Codec, Config, DataError, Deflater, Dictionary, Error, Inflater, LevelError, MemError, Reader, StreamError, UnsupportedError, Writer

Constant Summary

FASTEST =

:fastest

DEFAULT =

:default

BEST =

:best

DEFAULT_MAX_OUTPUT_SIZE =

512 * 1024 * 1024

DEFAULT_STREAMING_MAX_OUTPUT_SIZE =

2 * 1024 * 1024 * 1024

VERSION =

"0.6.0"

Class Method Summary

• .adler32(*args) ⇒ Object
• .algo_from_ext(path) ⇒ Object
• .algorithms ⇒ Object
• .available?(algo_sym) ⇒ Boolean
• .brotli(data, level: nil) ⇒ Object
• .brotli_decompress(data) ⇒ Object
• .compress(*args) ⇒ Object
• .config ⇒ Object
• .configure {|config| ... } ⇒ Object
• .crc32(*args) ⇒ Object
• .decompress(*args) ⇒ Object
• .lz4(data, level: nil, format: nil) ⇒ Object
• .lz4_decompress(data, format: nil) ⇒ Object
• .version(algo_sym) ⇒ Object
• .zstd(data, level: nil) ⇒ Object
• .zstd_decompress(data) ⇒ Object
• .zstd_frame_dictionary_id(data) ⇒ Object
• .zstd_single_frame_info(data) ⇒ Object

Class Method Details

.adler32(*args) ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 2052

static VALUE compress_adler32(int argc, VALUE *argv, VALUE self) {
    (void)self;
    VALUE data, prev;
    rb_scan_args(argc, argv, "11", &data, &prev);
    StringValue(data);

    const uint8_t *restrict src = (const uint8_t *)RSTRING_PTR(data);
    size_t len = (size_t)RSTRING_LEN(data);
    const uint32_t adler = NIL_P(prev) ? 1u : NUM2UINT(prev);

    uint32_t s1 = adler & 0xFFFFu;
    uint32_t s2 = (adler >> 16) & 0xFFFFu;
    enum { ADLER_BASE = 65521, ADLER_NMAX = 5552 };

    while (len > 0) {
        size_t chunk = len > ADLER_NMAX ? (size_t)ADLER_NMAX : len;
        len -= chunk;

        while (chunk >= 8) {
            s1 += src[0];
            s2 += s1;
            s1 += src[1];
            s2 += s1;
            s1 += src[2];
            s2 += s1;
            s1 += src[3];
            s2 += s1;
            s1 += src[4];
            s2 += s1;
            s1 += src[5];
            s2 += s1;
            s1 += src[6];
            s2 += s1;
            s1 += src[7];
            s2 += s1;
            src += 8;
            chunk -= 8;
        }
        while (chunk--) {
            s1 += *src++;
            s2 += s1;
        }
        s1 %= ADLER_BASE;
        s2 %= ADLER_BASE;
    }

    return UINT2NUM((s2 << 16) | s1);
}

.algo_from_ext(path) ⇒ Object

# File 'lib/multi_compress.rb', line 135

def self.algo_from_ext(path)
  EXTENSION_MAP[File.extname(path).downcase]
end

.algorithms ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 2101

static VALUE compress_algorithms(VALUE self) {
    VALUE ary = rb_ary_new_capa(3);
    rb_ary_push(ary, sym_cache.zstd);
    rb_ary_push(ary, sym_cache.lz4);
    rb_ary_push(ary, sym_cache.brotli);
    return ary;
}

.available?(algo_sym) ⇒ Boolean

Returns:

• (Boolean)

# File 'ext/multi_compress/multi_compress.c', line 2109

static VALUE compress_available_p(VALUE self, VALUE algo_sym) {
    sym_to_algo(algo_sym);
    return Qtrue;
}

.brotli(data, level: nil) ⇒ Object

# File 'lib/multi_compress.rb', line 111

def self.brotli(data, level: nil)
  compress(data, algo: :brotli, level: level)
end

.brotli_decompress(data) ⇒ Object

# File 'lib/multi_compress.rb', line 131

def self.brotli_decompress(data)
  decompress(data, algo: :brotli)
end

.compress(*args) ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 1259

static VALUE compress_compress(int argc, VALUE *argv, VALUE self) {
    VALUE data, opts;
    scan_one_required_keywords(argc, argv, &data, &opts);
    StringValue(data);

    mc_opts_t parsed_opts;
    mc_parse_opts(opts, &parsed_opts);
    reject_algorithm_keyword(&parsed_opts);

    VALUE algo_sym = parsed_opts.algo;
    VALUE level_val = parsed_opts.level;
    VALUE dict_val = parsed_opts.dictionary;

    int explicit_algo = !NIL_P(algo_sym);
    compress_algo_t algo = explicit_algo ? sym_to_algo(algo_sym) : ALGO_ZSTD;
    lz4_format_t lz4_format = parse_lz4_format(&parsed_opts, algo, explicit_algo);
    int level = resolve_level(algo, level_val);

    dictionary_t *dict = NULL;
    if (!NIL_P(dict_val)) {
        if (algo == ALGO_LZ4) {
            rb_raise(eUnsupportedError, "LZ4 does not support dictionaries");
        }
        dict = opt_dictionary(dict_val);
    }

    const char *src = RSTRING_PTR(data);
    size_t slen = RSTRING_LEN(data);
    const algo_policy_t *policy = algo_policy(algo);

    switch (algo) {
    case ALGO_ZSTD: {
        size_t bound = ZSTD_compressBound(slen);

        ZSTD_CDict *cdict = NULL;
        if (dict) {
            cdict = dict_get_cdict(dict, level);
            if (!cdict)
                rb_raise(eMemError, "zstd: failed to create/get cdict");
        }

        if (slen < policy->gvl_unlock_threshold) {
            VALUE dst = rb_binary_str_buf_reserve(bound);
            int ctx_error = 0;
            size_t csize =
                zstd_compress_cached(RSTRING_PTR(dst), bound, src, slen, level, cdict, &ctx_error);
            if (ctx_error)
                rb_raise(eMemError, "zstd: failed to create context");
            if (ZSTD_isError(csize))
                rb_raise(eError, "zstd compress: %s", ZSTD_getErrorName(csize));
            rb_str_set_len(dst, (long)csize);
            RB_GC_GUARD(data);
            RB_GC_GUARD(dict_val);
            return dst;
        }

        {
            VALUE scheduler = current_fiber_scheduler();
            work_exec_mode_t mode = select_fiber_nogvl_or_direct_mode(
                scheduler, slen, policy->gvl_unlock_threshold, policy->gvl_unlock_threshold);

            if (mode == WORK_EXEC_FIBER) {
                char *out_buf = (char *)malloc(bound);
                if (!out_buf)
                    rb_raise(eMemError, "zstd: malloc failed");
                zstd_fiber_compress_t fargs = {
                    .src = src,
                    .src_len = slen,
                    .level = level,
                    .cdict = cdict,
                    .dst = out_buf,
                    .dst_cap = bound,
                    .result = 0,
                    .error = 0,
                };

                RUN_WITH_EXEC_MODE(mode, zstd_fiber_compress_nogvl, fargs);

                if (fargs.error) {
                    free(out_buf);
                    rb_raise(eMemError, "zstd: failed to create context");
                }
                if (ZSTD_isError(fargs.result)) {
                    free(out_buf);
                    rb_raise(eError, "zstd compress: %s", ZSTD_getErrorName(fargs.result));
                }

                VALUE result = rb_binary_str_new(out_buf, (long)fargs.result);
                free(out_buf);
                RB_GC_GUARD(data);
                RB_GC_GUARD(dict_val);
                return result;
            }
        }

        {
            VALUE dst = rb_binary_str_buf_reserve(bound);
            zstd_compress_args_t args = {
                .src = src,
                .src_len = slen,
                .dst = RSTRING_PTR(dst),
                .dst_cap = bound,
                .level = level,
                .cdict = cdict,
                .result = 0,
                .error = 0,
            };
            RUN_WITH_EXEC_MODE(WORK_EXEC_NOGVL, zstd_compress_nogvl, args);

            if (args.error)
                rb_raise(eMemError, "zstd: failed to create context");
            if (ZSTD_isError(args.result))
                rb_raise(eError, "zstd compress: %s", ZSTD_getErrorName(args.result));

            rb_str_set_len(dst, (long)args.result);
            RB_GC_GUARD(data);
            RB_GC_GUARD(dict_val);
            return dst;
        }
    }
    case ALGO_LZ4: {
        if (lz4_format == LZ4_FORMAT_FRAME) {
            LZ4F_preferences_t prefs;
            memset(&prefs, 0, sizeof(prefs));
            prefs.frameInfo.blockChecksumFlag = LZ4F_blockChecksumEnabled;
            prefs.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled;
            size_t bound = LZ4F_compressFrameBound(slen, &prefs);
            VALUE dst = rb_binary_str_buf_reserve((long)bound);
            lz4frame_compress_args_t args = {
                .src = src,
                .src_len = slen,
                .dst = RSTRING_PTR(dst),
                .dst_cap = bound,
                .result = 0,
                .error_code = 0,
            };
            {
                VALUE scheduler = current_fiber_scheduler();
                work_exec_mode_t mode = select_fiber_nogvl_or_direct_mode(
                    scheduler, slen, policy->gvl_unlock_threshold, policy->gvl_unlock_threshold);
                RUN_WITH_EXEC_MODE(mode, lz4frame_compress_nogvl, args);
            }
            if (args.error_code)
                rb_raise(eError, "lz4 frame compress failed: %s",
                         LZ4F_getErrorName(args.error_code));
            rb_str_set_len(dst, (long)args.result);
            RB_GC_GUARD(data);
            return dst;
        }
        if (slen > (size_t)INT_MAX)
            rb_raise(eError, "lz4: input too large (max 2GB)");
        int bound = LZ4_compressBound((int)slen);

        int csize;
        if (slen >= policy->gvl_unlock_threshold) {
            VALUE dst = rb_binary_str_buf_reserve(8 + (size_t)bound + 4);
            char *out = RSTRING_PTR(dst);

            write_le_u32((uint8_t *)out, (uint32_t)slen);

            lz4_compress_args_t args = {
                .src = src,
                .src_len = (int)slen,
                .dst = out + 8,
                .dst_cap = bound,
                .level = level,
            };

            VALUE scheduler = current_fiber_scheduler();
            work_exec_mode_t mode = select_fiber_nogvl_or_direct_mode(
                scheduler, slen, policy->gvl_unlock_threshold, policy->gvl_unlock_threshold);
            RUN_WITH_EXEC_MODE(mode, lz4_compress_nogvl, args);
            csize = args.result;

            if (csize <= 0)
                rb_raise(eError, "lz4 compress failed");

            write_le_u32((uint8_t *)out + 4, (uint32_t)csize);

            size_t total = 8 + (size_t)csize;
            write_le_u32((uint8_t *)out + total, 0);

            rb_str_set_len(dst, (long)(total + 4));
            RB_GC_GUARD(data);
            return dst;
        } else {
            VALUE dst = rb_binary_str_buf_reserve(8 + bound + 4);
            char *out = RSTRING_PTR(dst);

            write_le_u32((uint8_t *)out, (uint32_t)slen);

            if (level > 1) {
                csize = LZ4_compress_HC(src, out + 8, (int)slen, bound, level);
            } else {
                csize = LZ4_compress_default(src, out + 8, (int)slen, bound);
            }
            if (csize <= 0)
                rb_raise(eError, "lz4 compress failed");

            write_le_u32((uint8_t *)out + 4, (uint32_t)csize);

            size_t total = 8 + csize;
            write_le_u32((uint8_t *)out + total, 0);

            rb_str_set_len(dst, total + 4);
            RB_GC_GUARD(data);
            return dst;
        }
    }
    case ALGO_BROTLI: {
        size_t out_len = BrotliEncoderMaxCompressedSize(slen);
        if (out_len == 0)
            out_len = slen + (slen >> 2) + 1024;

        if (dict) {
            VALUE dst = rb_binary_str_buf_reserve(out_len);
            BrotliEncoderState *enc = BrotliEncoderCreateInstance(NULL, NULL, NULL);
            if (!enc)
                rb_raise(eMemError, "brotli: failed to create encoder");

            BrotliEncoderPreparedDictionary *pd =
                BrotliEncoderPrepareDictionary(BROTLI_SHARED_DICTIONARY_RAW, dict->size, dict->data,
                                               BROTLI_MAX_QUALITY, NULL, NULL, NULL);
            if (!pd) {
                BrotliEncoderDestroyInstance(enc);
                rb_raise(eMemError, "brotli: failed to prepare dictionary");
            }

            if (!BrotliEncoderSetParameter(enc, BROTLI_PARAM_QUALITY, level)) {
                BrotliEncoderDestroyPreparedDictionary(pd);
                BrotliEncoderDestroyInstance(enc);
                rb_raise(eError, "brotli: failed to set quality parameter");
            }
            if (!BrotliEncoderSetParameter(enc, BROTLI_PARAM_SIZE_HINT,
                                           slen > UINT32_MAX ? UINT32_MAX : (uint32_t)slen)) {
                BrotliEncoderDestroyPreparedDictionary(pd);
                BrotliEncoderDestroyInstance(enc);
                rb_raise(eError, "brotli: failed to set size hint parameter");
            }
            if (!BrotliEncoderAttachPreparedDictionary(enc, pd)) {
                BrotliEncoderDestroyPreparedDictionary(pd);
                BrotliEncoderDestroyInstance(enc);
                rb_raise(eError, "brotli: failed to attach dictionary");
            }

            size_t available_in = slen;
            const uint8_t *next_in = (const uint8_t *)src;
            size_t available_out = out_len;
            uint8_t *next_out = (uint8_t *)RSTRING_PTR(dst);
            size_t initial_out = available_out;

            BROTLI_BOOL ok =
                BrotliEncoderCompressStream(enc, BROTLI_OPERATION_FINISH, &available_in, &next_in,
                                            &available_out, &next_out, NULL);

            BrotliEncoderDestroyPreparedDictionary(pd);
            BrotliEncoderDestroyInstance(enc);
            if (!ok)
                rb_raise(eError, "brotli compress with dict failed");

            rb_str_set_len(dst, initial_out - available_out);
            RB_GC_GUARD(data);
            RB_GC_GUARD(dict_val);
            return dst;
        } else if (slen >= policy->gvl_unlock_threshold) {
            VALUE dst = rb_binary_str_buf_reserve(out_len);
            size_t actual_out_len = out_len;

            brotli_compress_args_t args = {
                .level = level,
                .src_len = slen,
                .src = (const uint8_t *)src,
                .out_len = &actual_out_len,
                .dst = (uint8_t *)RSTRING_PTR(dst),
            };

            VALUE scheduler = current_fiber_scheduler();
            work_exec_mode_t mode = select_fiber_nogvl_or_direct_mode(
                scheduler, slen, policy->gvl_unlock_threshold, policy->gvl_unlock_threshold);
            RUN_WITH_EXEC_MODE(mode, brotli_compress_nogvl, args);

            if (!args.result)
                rb_raise(eError, "brotli compress failed");

            rb_str_set_len(dst, (long)actual_out_len);
            RB_GC_GUARD(data);
            return dst;
        } else {
            VALUE dst = rb_binary_str_buf_reserve(out_len);
            BROTLI_BOOL ok =
                BrotliEncoderCompress(level, BROTLI_DEFAULT_WINDOW, BROTLI_DEFAULT_MODE, slen,
                                      (const uint8_t *)src, &out_len, (uint8_t *)RSTRING_PTR(dst));
            if (!ok)
                rb_raise(eError, "brotli compress failed");
            rb_str_set_len(dst, out_len);
            RB_GC_GUARD(data);
            return dst;
        }
    }
    }

    return Qnil;
}

.config ⇒ Object

# File 'lib/multi_compress.rb', line 82

def config
  @config ||= Config.new
end

.configure {|config| ... } ⇒ Object

Yields:

• (config)

# File 'lib/multi_compress.rb', line 86

def configure
  return config unless block_given?

  yield(config)
  config
end

.crc32(*args) ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 2040

static VALUE compress_crc32(int argc, VALUE *argv, VALUE self) {
    VALUE data, prev;
    rb_scan_args(argc, argv, "11", &data, &prev);
    StringValue(data);

    const uint8_t *src = (const uint8_t *)RSTRING_PTR(data);
    size_t len = RSTRING_LEN(data);
    uint32_t crc = NIL_P(prev) ? 0 : NUM2UINT(prev);

    return UINT2NUM(crc32_compute(src, len, crc));
}

.decompress(*args) ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 1606

def self.decompress(data, **opts)
  _c_decompress(data, **resolved_one_shot_options(opts))
end

.lz4(data, level: nil, format: nil) ⇒ Object

# File 'lib/multi_compress.rb', line 103

def self.lz4(data, level: nil, format: nil)
  if format
    compress(data, algo: :lz4, level: level, format: format)
  else
    compress(data, algo: :lz4, level: level)
  end
end

.lz4_decompress(data, format: nil) ⇒ Object

# File 'lib/multi_compress.rb', line 123

def self.lz4_decompress(data, format: nil)
  if format
    decompress(data, algo: :lz4, format: format)
  else
    decompress(data, algo: :lz4)
  end
end

.version(algo_sym) ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 2114

static VALUE compress_version(VALUE self, VALUE algo_sym) {
    compress_algo_t algo = sym_to_algo(algo_sym);
    switch (algo) {
    case ALGO_ZSTD:
        return rb_str_new_cstr(ZSTD_versionString());
    case ALGO_LZ4:
        return rb_sprintf("%d.%d.%d", LZ4_VERSION_MAJOR, LZ4_VERSION_MINOR, LZ4_VERSION_RELEASE);
    case ALGO_BROTLI:
        return rb_sprintf("%d.%d.%d", BrotliEncoderVersion() >> 24,
                          (BrotliEncoderVersion() >> 12) & 0xFFF, BrotliEncoderVersion() & 0xFFF);
    }
    return Qnil;
}

.zstd(data, level: nil) ⇒ Object

# File 'lib/multi_compress.rb', line 99

def self.zstd(data, level: nil)
  compress(data, algo: :zstd, level: level)
end

.zstd_decompress(data) ⇒ Object

# File 'lib/multi_compress.rb', line 119

def self.zstd_decompress(data)
  decompress(data, algo: :zstd)
end

.zstd_frame_dictionary_id(data) ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 1563

static VALUE zstd_frame_dictionary_id(VALUE self, VALUE data) {
    (void)self;
    StringValue(data);

    const void *src = RSTRING_PTR(data);
    size_t src_len = (size_t)RSTRING_LEN(data);
    size_t frame_size = ZSTD_findFrameCompressedSize(src, src_len);
    if (ZSTD_isError(frame_size) || frame_size != src_len)
        rb_raise(eDataError, "zstd: expected exactly one valid frame with no trailing bytes");

    RB_GC_GUARD(data);
    return UINT2NUM(ZSTD_getDictID_fromFrame(src, src_len));
}

.zstd_single_frame_info(data) ⇒ Object

# File 'ext/multi_compress/multi_compress.c', line 1577

static VALUE zstd_single_frame_info(VALUE self, VALUE data) {
    (void)self;
    StringValue(data);

    const void *src = RSTRING_PTR(data);
    size_t src_len = (size_t)RSTRING_LEN(data);
    size_t frame_size = ZSTD_findFrameCompressedSize(src, src_len);
    if (ZSTD_isError(frame_size)) {
        rb_raise(eDataError, "zstd: invalid frame: %s", ZSTD_getErrorName(frame_size));
    }
    if (frame_size != src_len) {
        rb_raise(eDataError, "zstd: expected exactly one frame with no trailing bytes");
    }

    unsigned long long content_size = ZSTD_getFrameContentSize(src, src_len);
    if (content_size == ZSTD_CONTENTSIZE_ERROR) {
        rb_raise(eDataError, "zstd: invalid frame content size");
    }
    if (content_size == ZSTD_CONTENTSIZE_UNKNOWN) {
        rb_raise(eDataError, "zstd: frame must declare content size");
    }

    VALUE result = rb_ary_new_capa(2);
    rb_ary_push(result, SIZET2NUM(frame_size));
    rb_ary_push(result, ULL2NUM(content_size));
    RB_GC_GUARD(data);
    return result;
}