-
Notifications
You must be signed in to change notification settings - Fork 1
/
array.c
9047 lines (8247 loc) · 251 KB
/
array.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**********************************************************************
array.c -
$Author$
created at: Fri Aug 6 09:46:12 JST 1993
Copyright (C) 1993-2007 Yukihiro Matsumoto
Copyright (C) 2000 Network Applied Communication Laboratory, Inc.
Copyright (C) 2000 Information-technology Promotion Agency, Japan
**********************************************************************/
#include "debug_counter.h"
#include "id.h"
#include "internal.h"
#include "internal/array.h"
#include "internal/compar.h"
#include "internal/enum.h"
#include "internal/gc.h"
#include "internal/hash.h"
#include "internal/numeric.h"
#include "internal/object.h"
#include "internal/proc.h"
#include "internal/rational.h"
#include "internal/vm.h"
#include "probes.h"
#include "ruby/encoding.h"
#include "ruby/st.h"
#include "ruby/util.h"
#include "vm_core.h"
#include "builtin.h"
#if !ARRAY_DEBUG
# undef NDEBUG
# define NDEBUG
#endif
#include "ruby_assert.h"
// Conditional compilation macros for MMTk.
#include "internal/mmtk_macros.h"
VALUE rb_cArray;
VALUE rb_cArray_empty_frozen;
/* Flags of RArray
*
* 1: RARRAY_EMBED_FLAG
* The array is embedded (its contents follow the header, rather than
* being on a separately allocated buffer).
* 2: RARRAY_SHARED_FLAG (equal to ELTS_SHARED)
* The array is shared. The buffer this array points to is owned by
* another array (the shared root).
* 3-9: RARRAY_EMBED_LEN
* The length of the array when RARRAY_EMBED_FLAG is set.
* 12: RARRAY_SHARED_ROOT_FLAG
* The array is a shared root that does reference counting. The buffer
* this array points to is owned by this array but may be pointed to
* by other arrays.
* Note: Frozen arrays may be a shared root without this flag being
* set. Frozen arrays do not have reference counting because
* they cannot be modified. Not updating the reference count
* improves copy-on-write performance. Their reference count is
* assumed to be infinity.
* 14: RARRAY_PTR_IN_USE_FLAG
* The buffer of the array is in use. This is only used during
* debugging.
*/
/* for OPTIMIZED_CMP: */
#define id_cmp idCmp
#define ARY_DEFAULT_SIZE 16
#define ARY_MAX_SIZE (LONG_MAX / (int)sizeof(VALUE))
#define SMALL_ARRAY_LEN 16
RBIMPL_ATTR_MAYBE_UNUSED()
static int
should_be_T_ARRAY(VALUE ary)
{
return RB_TYPE_P(ary, T_ARRAY);
}
#define ARY_HEAP_PTR(a) (RUBY_ASSERT(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.ptr)
#define ARY_HEAP_LEN(a) (RUBY_ASSERT(!ARY_EMBED_P(a)), RARRAY(a)->as.heap.len)
#define ARY_HEAP_CAPA(a) (RUBY_ASSERT(!ARY_EMBED_P(a)), RUBY_ASSERT(!ARY_SHARED_ROOT_P(a)), \
RARRAY(a)->as.heap.aux.capa)
#define ARY_EMBED_PTR(a) (RUBY_ASSERT(ARY_EMBED_P(a)), RARRAY(a)->as.ary)
#define ARY_EMBED_LEN(a) \
(RUBY_ASSERT(ARY_EMBED_P(a)), \
(long)((RBASIC(a)->flags >> RARRAY_EMBED_LEN_SHIFT) & \
(RARRAY_EMBED_LEN_MASK >> RARRAY_EMBED_LEN_SHIFT)))
#define ARY_HEAP_SIZE(a) (RUBY_ASSERT(!ARY_EMBED_P(a)), RUBY_ASSERT(ARY_OWNS_HEAP_P(a)), ARY_CAPA(a) * sizeof(VALUE))
#define ARY_OWNS_HEAP_P(a) (RUBY_ASSERT(should_be_T_ARRAY((VALUE)(a))), \
!FL_TEST_RAW((a), RARRAY_SHARED_FLAG|RARRAY_EMBED_FLAG))
#define FL_SET_EMBED(a) do { \
RUBY_ASSERT(!ARY_SHARED_P(a)); \
FL_SET((a), RARRAY_EMBED_FLAG); \
ary_verify(a); \
} while (0)
#define FL_UNSET_EMBED(ary) FL_UNSET((ary), RARRAY_EMBED_FLAG|RARRAY_EMBED_LEN_MASK)
#define FL_SET_SHARED(ary) do { \
RUBY_ASSERT(!ARY_EMBED_P(ary)); \
FL_SET((ary), RARRAY_SHARED_FLAG); \
} while (0)
#define FL_UNSET_SHARED(ary) FL_UNSET((ary), RARRAY_SHARED_FLAG)
#define ARY_SET_PTR(ary, p) do { \
RUBY_ASSERT(!ARY_EMBED_P(ary)); \
RUBY_ASSERT(!OBJ_FROZEN(ary)); \
RARRAY(ary)->as.heap.ptr = (p); \
} while (0)
#define ARY_SET_EMBED_LEN(ary, n) do { \
long tmp_n = (n); \
RUBY_ASSERT(ARY_EMBED_P(ary)); \
RBASIC(ary)->flags &= ~RARRAY_EMBED_LEN_MASK; \
RBASIC(ary)->flags |= (tmp_n) << RARRAY_EMBED_LEN_SHIFT; \
} while (0)
#define ARY_SET_HEAP_LEN(ary, n) do { \
RUBY_ASSERT(!ARY_EMBED_P(ary)); \
RARRAY(ary)->as.heap.len = (n); \
} while (0)
#define ARY_SET_LEN(ary, n) do { \
if (ARY_EMBED_P(ary)) { \
ARY_SET_EMBED_LEN((ary), (n)); \
} \
else { \
ARY_SET_HEAP_LEN((ary), (n)); \
} \
RUBY_ASSERT(RARRAY_LEN(ary) == (n)); \
} while (0)
#define ARY_INCREASE_PTR(ary, n) do { \
RUBY_ASSERT(!ARY_EMBED_P(ary)); \
RUBY_ASSERT(!OBJ_FROZEN(ary)); \
RARRAY(ary)->as.heap.ptr += (n); \
} while (0)
#define ARY_INCREASE_LEN(ary, n) do { \
RUBY_ASSERT(!OBJ_FROZEN(ary)); \
if (ARY_EMBED_P(ary)) { \
ARY_SET_EMBED_LEN((ary), RARRAY_LEN(ary)+(n)); \
} \
else { \
RARRAY(ary)->as.heap.len += (n); \
} \
} while (0)
#define ARY_CAPA(ary) (ARY_EMBED_P(ary) ? ary_embed_capa(ary) : \
ARY_SHARED_ROOT_P(ary) ? RARRAY_LEN(ary) : ARY_HEAP_CAPA(ary))
#define ARY_SET_CAPA(ary, n) do { \
RUBY_ASSERT(!ARY_EMBED_P(ary)); \
RUBY_ASSERT(!ARY_SHARED_P(ary)); \
RUBY_ASSERT(!OBJ_FROZEN(ary)); \
RARRAY(ary)->as.heap.aux.capa = (n); \
} while (0)
#define ARY_SHARED_ROOT_OCCUPIED(ary) (!OBJ_FROZEN(ary) && ARY_SHARED_ROOT_REFCNT(ary) == 1)
#define ARY_SET_SHARED_ROOT_REFCNT(ary, value) do { \
RUBY_ASSERT(ARY_SHARED_ROOT_P(ary)); \
RUBY_ASSERT(!OBJ_FROZEN(ary)); \
RUBY_ASSERT((value) >= 0); \
RARRAY(ary)->as.heap.aux.capa = (value); \
} while (0)
#define FL_SET_SHARED_ROOT(ary) do { \
RUBY_ASSERT(!OBJ_FROZEN(ary)); \
RUBY_ASSERT(!ARY_EMBED_P(ary)); \
FL_SET((ary), RARRAY_SHARED_ROOT_FLAG); \
} while (0)
static inline void
ARY_SET(VALUE a, long i, VALUE v)
{
RUBY_ASSERT(!ARY_SHARED_P(a));
RUBY_ASSERT(!OBJ_FROZEN(a));
RARRAY_ASET(a, i, v);
}
#undef RARRAY_ASET
static long
ary_embed_capa(VALUE ary)
{
size_t size = rb_gc_obj_slot_size(ary) - offsetof(struct RArray, as.ary);
RUBY_ASSERT(size % sizeof(VALUE) == 0);
return size / sizeof(VALUE);
}
static size_t
ary_embed_size(long capa)
{
return offsetof(struct RArray, as.ary) + (sizeof(VALUE) * capa);
}
static bool
ary_embeddable_p(long capa)
{
return rb_gc_size_allocatable_p(ary_embed_size(capa));
}
bool
rb_ary_embeddable_p(VALUE ary)
{
/* An array cannot be turned embeddable when the array is:
* - Shared root: other objects may point to the buffer of this array
* so we cannot make it embedded.
* - Frozen: this array may also be a shared root without the shared root
* flag.
* - Shared: we don't want to re-embed an array that points to a shared
* root (to save memory).
*/
return !(ARY_SHARED_ROOT_P(ary) || OBJ_FROZEN(ary) || ARY_SHARED_P(ary));
}
size_t
rb_ary_size_as_embedded(VALUE ary)
{
size_t real_size;
if (ARY_EMBED_P(ary)) {
real_size = ary_embed_size(ARY_EMBED_LEN(ary));
}
else if (rb_ary_embeddable_p(ary)) {
real_size = ary_embed_size(ARY_HEAP_CAPA(ary));
}
else {
real_size = sizeof(struct RArray);
}
return real_size;
}
#if USE_MMTK
void
rb_mmtk_remember_array_content_holder(VALUE ary)
{
rb_gc_writebarrier_remember(rb_mmtk_array_content_holder(ary));
}
void
rb_mmtk_ary_set_objbuf(VALUE ary, VALUE objbuf)
{
RUBY_ASSERT(rb_mmtk_enabled_p());
RB_OBJ_WRITE(ary, &RARRAY_EXT(ary)->objbuf, objbuf);
}
void
rb_mmtk_ary_copy_objbuf_ref(VALUE dst_ary, VALUE src_ary)
{
RUBY_ASSERT(rb_mmtk_enabled_p());
RUBY_ASSERT(!ARY_EMBED_P(src_ary));
RUBY_ASSERT(!ARY_EMBED_P(dst_ary));
RUBY_ASSERT(!OBJ_FROZEN(dst_ary));
rb_mmtk_ary_set_objbuf(dst_ary, RARRAY_EXT(src_ary)->objbuf);
}
// Attach a heap array `ary` with a newly allocated imemo:mmtk_objbuf of the given capacity `capa`.
// The first `copy_len` elements of the new objbuf are copied from `src`, and `copy_len` must not
// exceed `capa`.
//
// `src` may point to an element of another heap object, in which case `src_obj` must point to the
// object into which `src` is pointed, and `src_obj` will be pinned during the execution of this
// function. If `src` does not point into another heap object, `src_obj` may be 0.
//
// Note: capa is the number of elements in the newly created buffer.
// copy_len is the number of elements to copy, not the number of bytes.
static inline void
rb_mmtk_ary_new_objbuf_copy(VALUE ary, size_t capa, VALUE src_obj, const VALUE *src, size_t copy_len)
{
RUBY_ASSERT(rb_mmtk_enabled_p());
// When using MMTk, as.heap.ptr points to the ary field of a rb_mmtk_objbuf_t
// which is allocated in the heap as an imemo:mmtk_objbuf.
rb_mmtk_objbuf_t *objbuf = rb_mmtk_new_objbuf(capa); // This may trigger GC, causing objects to be moved.
VALUE *elems = rb_mmtk_objbuf_to_elems(objbuf);
// Note that `ary` may be an existing array and `src` may point into `ary` or its existing
// buffer. Do not modify `ary` until the new strbuf is fully written.
if (src != NULL) {
RUBY_ASSERT(capa >= copy_len);
for (size_t i = 0; i < copy_len; i++) {
// TODO: use array copy write barrier after enabling StickyImmix.
elems[i] = src[i];
}
}
RARRAY(ary)->as.heap.ptr = elems;
rb_mmtk_ary_set_objbuf(ary, (VALUE)objbuf);
// Keep `src_obj` alive and pinned until the function exits.
RB_GC_GUARD(src_obj);
}
// Attach a heap array with a newly allocated empty imemo:mmtk_objbuf.
static inline void
rb_mmtk_ary_new_objbuf(VALUE ary, size_t capa)
{
rb_mmtk_ary_new_objbuf_copy(ary, capa, 0, NULL, 0);
}
#endif
#if ARRAY_DEBUG
#define ary_verify(ary) ary_verify_(ary, __FILE__, __LINE__)
static VALUE
ary_verify_(VALUE ary, const char *file, int line)
{
RUBY_ASSERT(RB_TYPE_P(ary, T_ARRAY));
if (ARY_SHARED_P(ary)) {
VALUE root = ARY_SHARED_ROOT(ary);
const VALUE *ptr = ARY_HEAP_PTR(ary);
const VALUE *root_ptr = RARRAY_CONST_PTR(root);
long len = ARY_HEAP_LEN(ary), root_len = RARRAY_LEN(root);
RUBY_ASSERT(ARY_SHARED_ROOT_P(root) || OBJ_FROZEN(root));
RUBY_ASSERT(root_ptr <= ptr && ptr + len <= root_ptr + root_len);
ary_verify(root);
}
else if (ARY_EMBED_P(ary)) {
RUBY_ASSERT(!ARY_SHARED_P(ary));
RUBY_ASSERT(RARRAY_LEN(ary) <= ary_embed_capa(ary));
}
else {
const VALUE *ptr = RARRAY_CONST_PTR(ary);
long i, len = RARRAY_LEN(ary);
volatile VALUE v;
if (len > 1) len = 1; /* check only HEAD */
for (i=0; i<len; i++) {
v = ptr[i]; /* access check */
}
v = v;
}
return ary;
}
void
rb_ary_verify(VALUE ary)
{
ary_verify(ary);
}
#else
#define ary_verify(ary) ((void)0)
#endif
VALUE *
rb_ary_ptr_use_start(VALUE ary)
{
#if ARRAY_DEBUG
FL_SET_RAW(ary, RARRAY_PTR_IN_USE_FLAG);
#endif
return (VALUE *)RARRAY_CONST_PTR(ary);
}
void
rb_ary_ptr_use_end(VALUE ary)
{
#if ARRAY_DEBUG
FL_UNSET_RAW(ary, RARRAY_PTR_IN_USE_FLAG);
#endif
}
void
rb_mem_clear(VALUE *mem, long size)
{
while (size--) {
*mem++ = Qnil;
}
}
static void
ary_mem_clear(VALUE ary, long beg, long size)
{
RARRAY_PTR_USE(ary, ptr, {
rb_mem_clear(ptr + beg, size);
});
}
static inline void
memfill(register VALUE *mem, register long size, register VALUE val)
{
while (size--) {
*mem++ = val;
}
}
static void
ary_memfill(VALUE ary, long beg, long size, VALUE val)
{
WHEN_USING_MMTK({
rb_mmtk_remember_array_content_holder(ary);
RARRAY_PTR_USE(ary, ptr, {
memfill(ptr + beg, size, val);
});
return;
})
RARRAY_PTR_USE(ary, ptr, {
memfill(ptr + beg, size, val);
RB_OBJ_WRITTEN(ary, Qundef, val);
});
}
static void
ary_memcpy0(VALUE ary, long beg, long argc, const VALUE *argv, VALUE buff_owner_ary)
{
RUBY_ASSERT(!ARY_SHARED_P(buff_owner_ary));
WHEN_USING_MMTK({
rb_mmtk_remember_array_content_holder(buff_owner_ary);
RARRAY_PTR_USE(ary, ptr, {
MEMCPY(ptr+beg, argv, VALUE, argc);
});
return;
})
if (argc > (int)(128/sizeof(VALUE)) /* is magic number (cache line size) */) {
rb_gc_writebarrier_remember(buff_owner_ary);
RARRAY_PTR_USE(ary, ptr, {
MEMCPY(ptr+beg, argv, VALUE, argc);
});
}
else {
int i;
RARRAY_PTR_USE(ary, ptr, {
for (i=0; i<argc; i++) {
RB_OBJ_WRITE(buff_owner_ary, &ptr[i+beg], argv[i]);
}
});
}
}
static void
ary_memcpy(VALUE ary, long beg, long argc, const VALUE *argv)
{
ary_memcpy0(ary, beg, argc, argv, ary);
}
static VALUE *
ary_heap_alloc_buffer(size_t capa)
{
#if USE_MMTK
if (rb_mmtk_enabled_p()) {
// rb_mmtk_ary_new_objbuf should be a drop-in replacement.
// But rb_mmtk_ary_new_objbuf_copy should be used when copying/reallocating/resizing.
rb_bug("ary_heap_alloc should not be called when using MMTk.");
}
#endif
return ALLOC_N(VALUE, capa);
}
static void
ary_heap_free_ptr(VALUE ary, const VALUE *ptr, long size)
{
#if USE_MMTK
if (rb_mmtk_enabled_p()) {
// When using MMTk, the underlying buffer is an imemo:mmtk_objbuf which will be GC-ed.
// We clear its objbuf field just to be safe.
rb_mmtk_ary_set_objbuf(ary, 0);
return;
}
#endif
ruby_sized_xfree((void *)ptr, size);
}
static void
ary_heap_free(VALUE ary)
{
#if USE_MMTK
if (rb_mmtk_enabled_p()) {
// When using MMTk, the underlying buffer is an imemo:mmtk_objbuf which will be GC-ed.
// We clear its objbuf field just to be safe.
rb_mmtk_ary_set_objbuf(ary, 0);
return;
}
#endif
ary_heap_free_ptr(ary, ARY_HEAP_PTR(ary), ARY_HEAP_SIZE(ary));
}
static size_t
ary_heap_realloc(VALUE ary, size_t new_capa)
{
#if USE_MMTK
if (!rb_mmtk_enabled_p()) {
#endif
RUBY_ASSERT(!OBJ_FROZEN(ary));
SIZED_REALLOC_N(RARRAY(ary)->as.heap.ptr, VALUE, new_capa, ARY_HEAP_CAPA(ary));
#if USE_MMTK
} else {
size_t old_capa = ARY_HEAP_CAPA(ary);
size_t copy_len = new_capa < old_capa ? new_capa : old_capa;
rb_mmtk_ary_new_objbuf_copy(ary, new_capa, RARRAY_EXT(ary)->objbuf, ARY_HEAP_PTR(ary), copy_len);
}
#endif
ary_verify(ary);
return new_capa;
}
void
rb_ary_make_embedded(VALUE ary)
{
WHEN_USING_MMTK({
rb_bug("rb_ary_make_embedded should not be called when using MMTk.");
})
RUBY_ASSERT(rb_ary_embeddable_p(ary));
if (!ARY_EMBED_P(ary)) {
const VALUE *buf = ARY_HEAP_PTR(ary);
long len = ARY_HEAP_LEN(ary);
FL_SET_EMBED(ary);
ARY_SET_EMBED_LEN(ary, len);
MEMCPY((void *)ARY_EMBED_PTR(ary), (void *)buf, VALUE, len);
ary_heap_free_ptr(ary, buf, len * sizeof(VALUE));
}
}
static void
ary_resize_capa(VALUE ary, long capacity)
{
RUBY_ASSERT(RARRAY_LEN(ary) <= capacity);
RUBY_ASSERT(!OBJ_FROZEN(ary));
RUBY_ASSERT(!ARY_SHARED_P(ary));
if (capacity > ary_embed_capa(ary)) {
size_t new_capa = capacity;
if (ARY_EMBED_P(ary)) {
long len = ARY_EMBED_LEN(ary);
#if USE_MMTK
if (!rb_mmtk_enabled_p()) {
#endif
VALUE *ptr = ary_heap_alloc_buffer(capacity);
MEMCPY(ptr, ARY_EMBED_PTR(ary), VALUE, len);
FL_UNSET_EMBED(ary);
ARY_SET_PTR(ary, ptr);
#if USE_MMTK
} else {
rb_mmtk_ary_new_objbuf_copy(ary, capacity, ary, ARY_EMBED_PTR(ary), len);
FL_UNSET_EMBED(ary);
}
#endif
ARY_SET_HEAP_LEN(ary, len);
}
else {
new_capa = ary_heap_realloc(ary, capacity);
}
ARY_SET_CAPA(ary, new_capa);
}
else {
if (!ARY_EMBED_P(ary)) {
long len = ARY_HEAP_LEN(ary);
long old_capa = ARY_HEAP_CAPA(ary);
const VALUE *ptr = ARY_HEAP_PTR(ary);
if (len > capacity) len = capacity;
MEMCPY((VALUE *)RARRAY(ary)->as.ary, ptr, VALUE, len);
ary_heap_free_ptr(ary, ptr, old_capa);
FL_SET_EMBED(ary);
ARY_SET_LEN(ary, len);
}
}
ary_verify(ary);
}
static inline void
ary_shrink_capa(VALUE ary)
{
long capacity = ARY_HEAP_LEN(ary);
long old_capa = ARY_HEAP_CAPA(ary);
RUBY_ASSERT(!ARY_SHARED_P(ary));
RUBY_ASSERT(old_capa >= capacity);
if (old_capa > capacity) {
size_t new_capa = ary_heap_realloc(ary, capacity);
ARY_SET_CAPA(ary, new_capa);
}
ary_verify(ary);
}
static void
ary_double_capa(VALUE ary, long min)
{
long new_capa = ARY_CAPA(ary) / 2;
if (new_capa < ARY_DEFAULT_SIZE) {
new_capa = ARY_DEFAULT_SIZE;
}
if (new_capa >= ARY_MAX_SIZE - min) {
new_capa = (ARY_MAX_SIZE - min) / 2;
}
new_capa += min;
ary_resize_capa(ary, new_capa);
ary_verify(ary);
}
static void
rb_ary_decrement_share(VALUE shared_root)
{
if (!OBJ_FROZEN(shared_root)) {
long num = ARY_SHARED_ROOT_REFCNT(shared_root);
ARY_SET_SHARED_ROOT_REFCNT(shared_root, num - 1);
}
}
static void
rb_ary_unshare(VALUE ary)
{
VALUE shared_root = ARY_SHARED_ROOT(ary);
rb_ary_decrement_share(shared_root);
FL_UNSET_SHARED(ary);
}
static void
rb_ary_reset(VALUE ary)
{
if (ARY_OWNS_HEAP_P(ary)) {
ary_heap_free(ary);
}
else if (ARY_SHARED_P(ary)) {
rb_ary_unshare(ary);
}
FL_SET_EMBED(ary);
ARY_SET_EMBED_LEN(ary, 0);
}
static VALUE
rb_ary_increment_share(VALUE shared_root)
{
if (!OBJ_FROZEN(shared_root)) {
long num = ARY_SHARED_ROOT_REFCNT(shared_root);
RUBY_ASSERT(num >= 0);
ARY_SET_SHARED_ROOT_REFCNT(shared_root, num + 1);
}
return shared_root;
}
static void
rb_ary_set_shared(VALUE ary, VALUE shared_root)
{
RUBY_ASSERT(!ARY_EMBED_P(ary));
RUBY_ASSERT(!OBJ_FROZEN(ary));
RUBY_ASSERT(ARY_SHARED_ROOT_P(shared_root) || OBJ_FROZEN(shared_root));
rb_ary_increment_share(shared_root);
FL_SET_SHARED(ary);
RB_OBJ_WRITE(ary, &RARRAY(ary)->as.heap.aux.shared_root, shared_root);
RB_DEBUG_COUNTER_INC(obj_ary_shared_create);
}
static inline void
rb_ary_modify_check(VALUE ary)
{
rb_check_frozen(ary);
ary_verify(ary);
}
void
rb_ary_cancel_sharing(VALUE ary)
{
if (ARY_SHARED_P(ary)) {
long shared_len, len = RARRAY_LEN(ary);
VALUE shared_root = ARY_SHARED_ROOT(ary);
ary_verify(shared_root);
if (len <= ary_embed_capa(ary)) {
const VALUE *ptr = ARY_HEAP_PTR(ary);
FL_UNSET_SHARED(ary);
FL_SET_EMBED(ary);
MEMCPY((VALUE *)ARY_EMBED_PTR(ary), ptr, VALUE, len);
rb_ary_decrement_share(shared_root);
ARY_SET_EMBED_LEN(ary, len);
}
else if (ARY_SHARED_ROOT_OCCUPIED(shared_root) && len > ((shared_len = RARRAY_LEN(shared_root))>>1)) {
long shift = RARRAY_CONST_PTR(ary) - RARRAY_CONST_PTR(shared_root);
FL_UNSET_SHARED(ary);
#if USE_MMTK
if (rb_mmtk_enabled_p()) {
// The objbuf of the current array must be the same as that of the shared root.
RUBY_ASSERT(RARRAY_EXT(ary)->objbuf == RARRAY_EXT(shared_root)->objbuf);
}
#endif
ARY_SET_PTR(ary, RARRAY_CONST_PTR(shared_root));
ARY_SET_CAPA(ary, shared_len);
RARRAY_PTR_USE(ary, ptr, {
MEMMOVE(ptr, ptr+shift, VALUE, len);
});
// MMTk: It looks like the following `FL_SET_EMBED` will effectively "kill" the
// `shared_root`. Its "embed length" was cleared in `FL_UNSET_EMBED` when
// `shared_root` became a "heap" array.
FL_SET_EMBED(shared_root);
rb_ary_decrement_share(shared_root);
}
else {
#if USE_MMTK
if (!rb_mmtk_enabled_p()) {
#endif
VALUE *ptr = ary_heap_alloc_buffer(len);
MEMCPY(ptr, ARY_HEAP_PTR(ary), VALUE, len);
ARY_SET_PTR(ary, ptr);
#if USE_MMTK
} else {
rb_mmtk_ary_new_objbuf_copy(ary, len, RARRAY_EXT(ary)->objbuf, ARY_HEAP_PTR(ary), len);
}
#endif
rb_ary_unshare(ary);
ARY_SET_CAPA(ary, len);
}
WHEN_USING_MMTK2({
rb_mmtk_remember_array_content_holder(ary);
}, { // when not using MMTk
rb_gc_writebarrier_remember(ary);
})
}
ary_verify(ary);
}
void
rb_ary_modify(VALUE ary)
{
rb_ary_modify_check(ary);
rb_ary_cancel_sharing(ary);
}
static VALUE
ary_ensure_room_for_push(VALUE ary, long add_len)
{
long old_len = RARRAY_LEN(ary);
long new_len = old_len + add_len;
long capa;
if (old_len > ARY_MAX_SIZE - add_len) {
rb_raise(rb_eIndexError, "index %ld too big", new_len);
}
if (ARY_SHARED_P(ary)) {
if (new_len > ary_embed_capa(ary)) {
VALUE shared_root = ARY_SHARED_ROOT(ary);
if (ARY_SHARED_ROOT_OCCUPIED(shared_root)) {
if (ARY_HEAP_PTR(ary) - RARRAY_CONST_PTR(shared_root) + new_len <= RARRAY_LEN(shared_root)) {
rb_ary_modify_check(ary);
ary_verify(ary);
ary_verify(shared_root);
return shared_root;
}
else {
/* if array is shared, then it is likely it participate in push/shift pattern */
rb_ary_modify(ary);
capa = ARY_CAPA(ary);
if (new_len > capa - (capa >> 6)) {
ary_double_capa(ary, new_len);
}
ary_verify(ary);
return ary;
}
}
}
ary_verify(ary);
rb_ary_modify(ary);
}
else {
rb_ary_modify_check(ary);
}
capa = ARY_CAPA(ary);
if (new_len > capa) {
ary_double_capa(ary, new_len);
}
ary_verify(ary);
return ary;
}
/*
* call-seq:
* freeze -> self
*
* Freezes +self+ (if not already frozen); returns +self+:
*
* a = []
* a.frozen? # => false
* a.freeze
* a.frozen? # => true
*
* No further changes may be made to +self+;
* raises FrozenError if a change is attempted.
*
* Related: Kernel#frozen?.
*/
VALUE
rb_ary_freeze(VALUE ary)
{
RUBY_ASSERT(RB_TYPE_P(ary, T_ARRAY));
if (OBJ_FROZEN(ary)) return ary;
if (!ARY_EMBED_P(ary) && !ARY_SHARED_P(ary) && !ARY_SHARED_ROOT_P(ary)) {
ary_shrink_capa(ary);
}
return rb_obj_freeze(ary);
}
/* This can be used to take a snapshot of an array (with
e.g. rb_ary_replace) and check later whether the array has been
modified from the snapshot. The snapshot is cheap, though if
something does modify the array it will pay the cost of copying
it. If Array#pop or Array#shift has been called, the array will
be still shared with the snapshot, but the array length will
differ. */
VALUE
rb_ary_shared_with_p(VALUE ary1, VALUE ary2)
{
if (!ARY_EMBED_P(ary1) && ARY_SHARED_P(ary1) &&
!ARY_EMBED_P(ary2) && ARY_SHARED_P(ary2) &&
ARY_SHARED_ROOT(ary1) == ARY_SHARED_ROOT(ary2) &&
ARY_HEAP_LEN(ary1) == ARY_HEAP_LEN(ary2)) {
return Qtrue;
}
return Qfalse;
}
#if USE_MMTK
// How large is the array allocated with ary_alloc_heap
static inline size_t
rb_mmtk_ary_heap_size(void)
{
// The main RArray plus the arrayext.
return sizeof(struct RArray) + sizeof(rb_mmtk_arrayext_t);
}
#endif
static VALUE
ary_alloc_embed(VALUE klass, long capa)
{
size_t size = ary_embed_size(capa);
#if USE_MMTK
if (rb_mmtk_enabled_p()) {
if (size < rb_mmtk_ary_heap_size()) {
// When using MMTk, we always allocate enough space to hold a heap array.
// The lowest size class for vanilla Ruby gc is 40 bytes,
// which is enough to hold a whole `struct RArray` for heap arrays.
// But we have one extra field in the trailing rb_mmtk_arrayext_t.
// So we manually ensure the allocated memory region is large enough.
size = rb_mmtk_ary_heap_size();
}
}
#endif
RUBY_ASSERT(rb_gc_size_allocatable_p(size));
NEWOBJ_OF(ary, struct RArray, klass,
T_ARRAY | RARRAY_EMBED_FLAG | (RGENGC_WB_PROTECTED_ARRAY ? FL_WB_PROTECTED : 0),
size, 0);
/* Created array is:
* FL_SET_EMBED((VALUE)ary);
* ARY_SET_EMBED_LEN((VALUE)ary, 0);
*/
return (VALUE)ary;
}
static VALUE
ary_alloc_heap(VALUE klass)
{
size_t size = sizeof(struct RString);
#if USE_MMTK
if (rb_mmtk_enabled_p()) {
// When using MMTk, we include a trailing rb_mmtk_arrayext_t.
size = rb_mmtk_ary_heap_size();
}
#endif
NEWOBJ_OF(ary, struct RArray, klass,
T_ARRAY | (RGENGC_WB_PROTECTED_ARRAY ? FL_WB_PROTECTED : 0),
size, 0);
return (VALUE)ary;
}
static VALUE
empty_ary_alloc(VALUE klass)
{
RUBY_DTRACE_CREATE_HOOK(ARRAY, 0);
return ary_alloc_embed(klass, 0);
}
static VALUE
ary_new(VALUE klass, long capa)
{
VALUE ary;
if (capa < 0) {
rb_raise(rb_eArgError, "negative array size (or size too big)");
}
if (capa > ARY_MAX_SIZE) {
rb_raise(rb_eArgError, "array size too big");
}
RUBY_DTRACE_CREATE_HOOK(ARRAY, capa);
if (ary_embeddable_p(capa)) {
ary = ary_alloc_embed(klass, capa);
}
else {
ary = ary_alloc_heap(klass);
ARY_SET_CAPA(ary, capa);
RUBY_ASSERT(!ARY_EMBED_P(ary));
#if USE_MMTK
if (!rb_mmtk_enabled_p()) {
#endif
ARY_SET_PTR(ary, ary_heap_alloc_buffer(capa));
#if USE_MMTK
} else {
rb_mmtk_ary_new_objbuf(ary, capa);
}
#endif
ARY_SET_HEAP_LEN(ary, 0);
}
return ary;
}
VALUE
rb_ary_new_capa(long capa)
{
return ary_new(rb_cArray, capa);
}
VALUE
rb_ary_new(void)
{
return rb_ary_new_capa(0);
}
VALUE
(rb_ary_new_from_args)(long n, ...)
{
va_list ar;
VALUE ary;
long i;
ary = rb_ary_new2(n);
va_start(ar, n);
for (i=0; i<n; i++) {
ARY_SET(ary, i, va_arg(ar, VALUE));
}
va_end(ar);
ARY_SET_LEN(ary, n);
return ary;
}
VALUE
rb_ary_tmp_new_from_values(VALUE klass, long n, const VALUE *elts)
{
VALUE ary;
ary = ary_new(klass, n);
if (n > 0 && elts) {
ary_memcpy(ary, 0, n, elts);
ARY_SET_LEN(ary, n);
}
return ary;
}
VALUE
rb_ary_new_from_values(long n, const VALUE *elts)
{
return rb_ary_tmp_new_from_values(rb_cArray, n, elts);
}
static VALUE
ec_ary_alloc_embed(rb_execution_context_t *ec, VALUE klass, long capa)
{
#if USE_MMTK
if (rb_mmtk_enabled_p()) {
// The optimization about ec is unnecessary for MMTk. We avoid code duplication.
return ary_alloc_embed(klass, capa);
}
#endif
size_t size = ary_embed_size(capa);