Class: JSON::ResumableParser

Inherits:
Object
  • Object
show all
Defined in:
ext/json/ext/parser/parser.c,
lib/json/ext.rb

Instance Method Summary collapse

Instance Method Details

#<<(string) ⇒ self

Appends the given string to the parser's buffer.

Returns:

  • (self) 


2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
 
# File 'ext/json/ext/parser/parser.c', line 2410

static VALUE cResumableParser_feed(VALUE self, VALUE str)
{
    rb_check_frozen(self);

    JSON_ResumableParser *parser = ResumableParser_acquire(self, false);

    str = convert_encoding(str);
    if (!RSTRING_LEN(str)) {
        return self;
    }

    size_t offset = parser->state.cursor - parser->state.start;
    const size_t remaining = parser->state.end - parser->state.cursor;

    if (!remaining) {
        if (parser->buffer) {
            json_str_clear(parser->buffer);
        }
        parser->buffer = RB_OBJ_FROZEN_RAW(str) ? str : rb_obj_hide(rb_str_new_shared(str));
        offset = 0;
    } else {
        JSON_ASSERT(parser->buffer);

        const size_t size = parser->state.end - parser->state.start;
        const size_t consumed = size - remaining;

        if (RB_OBJ_FROZEN_RAW(parser->buffer)) {
            VALUE new_buffer = rb_obj_hide(rb_str_buf_new(remaining + RSTRING_LEN(str)));
            rb_enc_associate_index(new_buffer, utf8_encindex);

            char *old_ptr = RSTRING_PTR(parser->buffer);
            memcpy(RSTRING_PTR(new_buffer), old_ptr + consumed, remaining);
            rb_str_set_len(new_buffer, remaining);
            offset = 0;
            parser->buffer = new_buffer;
        } else if (consumed > (size / 2) && size >= 512) {
            rb_str_modify(parser->buffer);
            char *old_ptr = RSTRING_PTR(parser->buffer);
            memmove(old_ptr, old_ptr + consumed, remaining);
            rb_str_set_len(parser->buffer, remaining);
            offset = 0;
        }
        rb_str_append(parser->buffer, str);
    }

    long len;
    const char *start;
    RSTRING_GETMEM(parser->buffer, start, len);
    parser->state.start = start;
    parser->state.end = start + len;
    parser->state.cursor = parser->state.start + offset;

    return self;
}

#clear ⇒ self

Entirely reset the parser state and buffer.

Returns:

  • (self) 


2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
 
# File 'ext/json/ext/parser/parser.c', line 2630

static VALUE cResumableParser_clear(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, false);
    parser->buffer = 0;
    parser->complete = true;
    parser->parsed_bytes = 0;
    parser->incomplete_bytes = 0;
    parser->frames.head = 0;
    parser->value_stack.head = 0;
    parser->state.name_cache.length = 0;
    parser->state.current_nesting = 0;
    parser->state.in_array = 1;
    parser->state.emitted_deprecations = 0;
    parser->state.start = parser->state.cursor = parser->state.end = NULL;
    return self;
}

#empty? ⇒ Boolean

Returns whether the parser is entirely done: no unconsumed bytes in the buffer, no document under construction and no parsed value awaiting retrieval.

The main use case is detecting a truncated stream once the input is exhausted:

loop do
begin
  parser << socket.readpartial(4096)
rescue EOFError
  break
end
while parser.parse
  process(parser.value)
end
end
warn "stream was truncated" unless parser.empty?

Returns:

  • (Boolean) 


64
65
66
 
# File 'lib/json/ext.rb', line 64

def empty?
  eos? && !partial_value? && !value?
end

#eos? ⇒ Boolean

Returns whether the internal buffer has been entirely consumed.

Returns:

  • (Boolean) 


2771
2772
2773
2774
2775
 
# File 'ext/json/ext/parser/parser.c', line 2771

static VALUE cResumableParser_eos_p(VALUE self)
{
    JSON_ResumableParser *parser = cResumableParser_get(self);
    return eos(&parser->state) ? Qtrue : Qfalse;
}

#parse ⇒ Boolean

Attemps to parse a JSON document from the internal buffer. Returns whether a complete document could be parsed.

It does raise JSON::ParserError when encountering invalid JSON syntax.

The parsed object can be retrieved by calling #value

Returns:

  • (Boolean) 


2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
 
# File 'ext/json/ext/parser/parser.c', line 2515

static VALUE cResumableParser_parse(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, true);

    if (parser->complete) {
        parser->parsed_bytes = 0;
        parser->incomplete_bytes = 0;
        parser->complete = false;
    }

    if (!parser->buffer) {
        parser->in_use = false;
        return Qfalse;
    }

    if (parser->frames.head == 0) {
        json_frame_stack_push(&parser->state, (json_frame){
            .type = JSON_FRAME_ROOT,
            .phase = JSON_PHASE_VALUE,
        });
    }

    VALUE Vsource = parser->buffer; // Prevent compaction

    json_frame *frame = json_frame_stack_peek(&parser->frames);

    if (frame->phase == JSON_PHASE_DONE) {
        JSON_ASSERT(parser->value_stack.head == 1);
        JSON_ASSERT(parser->frames.head == 1);

        frame->phase = JSON_PHASE_VALUE;
        rvalue_stack_pop(parser->state.value_stack, 1);
    }

    struct json_parse_any_args args = {
        .state = &parser->state,
        .config = &parser->config,
        .parser = self,
    };
    int status;
    const char *initial_cursor = parser->state.cursor;
    parser->complete = rb_protect(json_parse_any_resumable_safe, (VALUE)&args, &status);

    if (status) {
        parser->complete = true; // a parse error is considered complete
    }

    parser->parsed_bytes += parser->state.cursor - initial_cursor;
    parser->incomplete_bytes = parser->complete ? 0 : parser->state.end - parser->state.cursor;

    json_eat_whitespace(&parser->state, &parser->config, false);
    if (eos(&parser->state)) {
        json_str_clear(parser->buffer);
        parser->buffer = Qfalse;
    }
    parser->in_use = false;

    if (status) {
        rb_jump_tag(status); // reraise
    }
    RB_GC_GUARD(Vsource);
    return parser->complete ? Qtrue : Qfalse;
}

#parsed_bytes ⇒ Integer

Returns the number of bytes parsed since the start of the current partial value. This is intended to be used for securing against untrusted input:

loop do if parser.parsed_bytes > DOCUMENT_MAX_SIZE raise "document too large" end

parser << read_chunk
while parser.parse
 process(parser.value)
end

end

Returns:

  • (Integer) 


2829
2830
2831
2832
2833
 
# File 'ext/json/ext/parser/parser.c', line 2829

static VALUE cResumableParser_parsed_bytes(VALUE self)
{
    JSON_ResumableParser *parser = cResumableParser_get(self);
    return ULL2NUM(parser->parsed_bytes + parser->incomplete_bytes);
}

#partial_value ⇒ Object

Returns the Ruby objects parsed up to this point:

parser << '[1, [2, 3,'
parser.parse # => false
parser.value # ArgumentError no ready value
parser.partial_value # => [1, [2, 3]]

Returns:



2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
 
# File 'ext/json/ext/parser/parser.c', line 2730

static VALUE cResumableParser_partial_value(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, true);

    int status;
    VALUE result = rb_protect(cResumableParser_partial_value_body, self, &status);
    parser->in_use = false;
    if (status) {
        rb_jump_tag(status);
    }
    return result;
}

#partial_value? ⇒ Boolean

Returns whether a document is currently under construction: an unclosed container, a key awaiting its value, etc.

It answers the same question as !partial_value.nil?, but as a cheap predicate on the parser's internal state, without materializing the partially parsed Ruby objects:

parser << '{"a":1,'
parser.parse # => false
parser.partial_value? # => true

A fully parsed document whose value hasn't been retrieved yet is not under construction: #value? returns true and #partial_value? returns false.

Returns:

  • (Boolean) 


2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
 
# File 'ext/json/ext/parser/parser.c', line 2793

static VALUE cResumableParser_partial_value_p(VALUE self)
{
    JSON_ResumableParser *parser = cResumableParser_get(self);

    // Mirror of #value?: values on the stack while the document isn't DONE
    // belong to a partially built document. A container whose first key or
    // element hasn't been parsed yet has no frame nor value registered (the
    // tokenizer rewinds to the container start on EOS), so that state is
    // observable through the buffer (#eos?/#rest) instead, keeping this
    // predicate consistent with #partial_value returning nil.
    if (parser->value_stack.head > 0) {
        json_frame *frame = json_frame_stack_peek(&parser->frames);
        if (frame->phase != JSON_PHASE_DONE) {
            return Qtrue;
        }
    }
    return Qfalse;
}

#rest ⇒ String

Returns a string containing what remains to be parsed in the buffer parser << '{ "message": "unterminated message' parser.parse # => false parser.rest # => '"unterminated message"'

Returns:



2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
 
# File 'ext/json/ext/parser/parser.c', line 2751

static VALUE cResumableParser_rest(VALUE self)
{
    JSON_ResumableParser *parser = cResumableParser_get(self);

    if (!parser->buffer) {
        return rb_utf8_str_new("", 0);
    }

    size_t offset = parser->state.cursor - parser->state.start;
    const char *ptr;
    long len;
    RSTRING_GETMEM(parser->buffer, ptr, len);
    return rb_utf8_str_new(ptr + offset, len - offset);
}

#value ⇒ Object

Returns and consume the last parsed value. Raises ArgumentError if there is no parsed value or if it was already retrieved:

parser << '[1][2]'
parser.value # ArgumentError no ready value
parser.parse # => true
parser.value # => [1]
parser.value # ArgumentError no ready value

Returns:



2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
 
# File 'ext/json/ext/parser/parser.c', line 2608

static VALUE cResumableParser_value(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, false);

    if (parser->frames.head > 0) {
        json_frame *frame = json_frame_stack_peek(&parser->frames);

        if (frame->phase == JSON_PHASE_DONE) {
            VALUE result = *rvalue_stack_peek(parser->state.value_stack, 1);
            rvalue_stack_pop(parser->state.value_stack, 1);
            json_frame_stack_pop(parser->state.frames);
            return result;
        }
    }
    rb_raise(rb_eArgError, "no ready value");
}

#value? ⇒ Boolean

Returns whether a parsed value is available.

Returns:

  • (Boolean) 


2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
 
# File 'ext/json/ext/parser/parser.c', line 2584

static VALUE cResumableParser_value_p(VALUE self)
{
    JSON_ResumableParser *parser = ResumableParser_acquire(self, false);

    if (parser->value_stack.head > 0) {
        json_frame *frame = json_frame_stack_peek(&parser->frames);
        if (frame->phase == JSON_PHASE_DONE) {
            return Qtrue;
        }
    }
    return Qfalse;
}