1 /*
2 +----------------------------------------------------------------------+
3 | Zend Engine |
4 +----------------------------------------------------------------------+
5 | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 2.00 of the Zend license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.zend.com/license/2_00.txt. |
11 | If you did not receive a copy of the Zend license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@zend.com so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Andi Gutmans <andi@php.net> |
16 | Zeev Suraski <zeev@php.net> |
17 | Dmitry Stogov <dmitry@php.net> |
18 +----------------------------------------------------------------------+
19 */
20
21 /*
22 * zend_alloc is designed to be a modern CPU cache friendly memory manager
23 * for PHP. Most ideas are taken from jemalloc and tcmalloc implementations.
24 *
25 * All allocations are split into 3 categories:
26 *
27 * Huge - the size is greater than CHUNK size (~2M by default), allocation is
28 * performed using mmap(). The result is aligned on 2M boundary.
29 *
30 * Large - a number of 4096K pages inside a CHUNK. Large blocks
31 * are always aligned on page boundary.
32 *
33 * Small - less than 3/4 of page size. Small sizes are rounded up to nearest
34 * greater predefined small size (there are 30 predefined sizes:
35 * 8, 16, 24, 32, ... 3072). Small blocks are allocated from
36 * RUNs. Each RUN is allocated as a single or few following pages.
37 * Allocation inside RUNs implemented using linked list of free
38 * elements. The result is aligned to 8 bytes.
39 *
40 * zend_alloc allocates memory from OS by CHUNKs, these CHUNKs and huge memory
41 * blocks are always aligned to CHUNK boundary. So it's very easy to determine
42 * the CHUNK owning the certain pointer. Regular CHUNKs reserve a single
43 * page at start for special purpose. It contains bitset of free pages,
44 * few bitset for available runs of predefined small sizes, map of pages that
45 * keeps information about usage of each page in this CHUNK, etc.
46 *
47 * zend_alloc provides familiar emalloc/efree/erealloc API, but in addition it
48 * provides specialized and optimized routines to allocate blocks of predefined
49 * sizes (e.g. emalloc_2(), emallc_4(), ..., emalloc_large(), etc)
50 * The library uses C preprocessor tricks that substitute calls to emalloc()
51 * with more specialized routines when the requested size is known.
52 */
53
54 #include "zend.h"
55 #include "zend_alloc.h"
56 #include "zend_globals.h"
57 #include "zend_operators.h"
58 #include "zend_multiply.h"
59 #include "zend_bitset.h"
60 #include "zend_mmap.h"
61 #include <signal.h>
62
63 #ifdef HAVE_UNISTD_H
64 # include <unistd.h>
65 #endif
66
67 #ifdef ZEND_WIN32
68 # include <wincrypt.h>
69 # include <process.h>
70 # include "win32/winutil.h"
71 #endif
72
73 #include <stdio.h>
74 #include <stdlib.h>
75 #include <string.h>
76
77 #include <sys/types.h>
78 #include <sys/stat.h>
79 #include <limits.h>
80 #include <fcntl.h>
81 #include <errno.h>
82 #ifdef __SANITIZE_ADDRESS__
83 # include <sanitizer/asan_interface.h>
84 #endif
85
86 #ifndef _WIN32
87 # include <sys/mman.h>
88 # ifndef MAP_ANON
89 # ifdef MAP_ANONYMOUS
90 # define MAP_ANON MAP_ANONYMOUS
91 # endif
92 # endif
93 # ifndef MAP_FAILED
94 # define MAP_FAILED ((void*)-1)
95 # endif
96 # ifndef MAP_POPULATE
97 # define MAP_POPULATE 0
98 # endif
99 # if defined(_SC_PAGESIZE) || (_SC_PAGE_SIZE)
100 # define REAL_PAGE_SIZE _real_page_size
101 static size_t _real_page_size = ZEND_MM_PAGE_SIZE;
102 # endif
103 # ifdef MAP_ALIGNED_SUPER
104 # define MAP_HUGETLB MAP_ALIGNED_SUPER
105 # endif
106 #endif
107
108 #ifndef REAL_PAGE_SIZE
109 # define REAL_PAGE_SIZE ZEND_MM_PAGE_SIZE
110 #endif
111
112 /* NetBSD has an mremap() function with a signature that is incompatible with Linux (WTF?),
113 * so pretend it doesn't exist. */
114 #ifndef __linux__
115 # undef HAVE_MREMAP
116 #endif
117
118 #ifndef __APPLE__
119 # define ZEND_MM_FD -1
120 #else
121 # include <mach/vm_statistics.h>
122 /* Mac allows to track anonymous page via vmmap per TAG id.
123 * user land applications are allowed to take from 240 to 255.
124 */
125 # define ZEND_MM_FD VM_MAKE_TAG(250U)
126 #endif
127
128 #ifndef ZEND_MM_STAT
129 # define ZEND_MM_STAT 1 /* track current and peak memory usage */
130 #endif
131 #ifndef ZEND_MM_LIMIT
132 # define ZEND_MM_LIMIT 1 /* support for user-defined memory limit */
133 #endif
134 #ifndef ZEND_MM_CUSTOM
135 # define ZEND_MM_CUSTOM 1 /* support for custom memory allocator */
136 /* USE_ZEND_ALLOC=0 may switch to system malloc() */
137 #endif
138 #ifndef ZEND_MM_STORAGE
139 # define ZEND_MM_STORAGE 1 /* support for custom memory storage */
140 #endif
141 #ifndef ZEND_MM_ERROR
142 # define ZEND_MM_ERROR 1 /* report system errors */
143 #endif
144
145 #ifndef ZEND_MM_CHECK
146 # define ZEND_MM_CHECK(condition, message) do { \
147 if (UNEXPECTED(!(condition))) { \
148 zend_mm_panic(message); \
149 } \
150 } while (0)
151 #endif
152
153 typedef uint32_t zend_mm_page_info; /* 4-byte integer */
154 typedef zend_ulong zend_mm_bitset; /* 4-byte or 8-byte integer */
155
156 #define ZEND_MM_ALIGNED_OFFSET(size, alignment) \
157 (((size_t)(size)) & ((alignment) - 1))
158 #define ZEND_MM_ALIGNED_BASE(size, alignment) \
159 (((size_t)(size)) & ~((alignment) - 1))
160 #define ZEND_MM_SIZE_TO_NUM(size, alignment) \
161 (((size_t)(size) + ((alignment) - 1)) / (alignment))
162
163 #define ZEND_MM_BITSET_LEN (sizeof(zend_mm_bitset) * 8) /* 32 or 64 */
164 #define ZEND_MM_PAGE_MAP_LEN (ZEND_MM_PAGES / ZEND_MM_BITSET_LEN) /* 16 or 8 */
165
166 typedef zend_mm_bitset zend_mm_page_map[ZEND_MM_PAGE_MAP_LEN]; /* 64B */
167
168 #define ZEND_MM_IS_FRUN 0x00000000
169 #define ZEND_MM_IS_LRUN 0x40000000
170 #define ZEND_MM_IS_SRUN 0x80000000
171
172 #define ZEND_MM_LRUN_PAGES_MASK 0x000003ff
173 #define ZEND_MM_LRUN_PAGES_OFFSET 0
174
175 #define ZEND_MM_SRUN_BIN_NUM_MASK 0x0000001f
176 #define ZEND_MM_SRUN_BIN_NUM_OFFSET 0
177
178 #define ZEND_MM_SRUN_FREE_COUNTER_MASK 0x01ff0000
179 #define ZEND_MM_SRUN_FREE_COUNTER_OFFSET 16
180
181 #define ZEND_MM_NRUN_OFFSET_MASK 0x01ff0000
182 #define ZEND_MM_NRUN_OFFSET_OFFSET 16
183
184 #define ZEND_MM_LRUN_PAGES(info) (((info) & ZEND_MM_LRUN_PAGES_MASK) >> ZEND_MM_LRUN_PAGES_OFFSET)
185 #define ZEND_MM_SRUN_BIN_NUM(info) (((info) & ZEND_MM_SRUN_BIN_NUM_MASK) >> ZEND_MM_SRUN_BIN_NUM_OFFSET)
186 #define ZEND_MM_SRUN_FREE_COUNTER(info) (((info) & ZEND_MM_SRUN_FREE_COUNTER_MASK) >> ZEND_MM_SRUN_FREE_COUNTER_OFFSET)
187 #define ZEND_MM_NRUN_OFFSET(info) (((info) & ZEND_MM_NRUN_OFFSET_MASK) >> ZEND_MM_NRUN_OFFSET_OFFSET)
188
189 #define ZEND_MM_FRUN() ZEND_MM_IS_FRUN
190 #define ZEND_MM_LRUN(count) (ZEND_MM_IS_LRUN | ((count) << ZEND_MM_LRUN_PAGES_OFFSET))
191 #define ZEND_MM_SRUN(bin_num) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET))
192 #define ZEND_MM_SRUN_EX(bin_num, count) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((count) << ZEND_MM_SRUN_FREE_COUNTER_OFFSET))
193 #define ZEND_MM_NRUN(bin_num, offset) (ZEND_MM_IS_SRUN | ZEND_MM_IS_LRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((offset) << ZEND_MM_NRUN_OFFSET_OFFSET))
194
195 #define ZEND_MM_BINS 30
196
197 typedef struct _zend_mm_page zend_mm_page;
198 typedef struct _zend_mm_bin zend_mm_bin;
199 typedef struct _zend_mm_free_slot zend_mm_free_slot;
200 typedef struct _zend_mm_chunk zend_mm_chunk;
201 typedef struct _zend_mm_huge_list zend_mm_huge_list;
202
203 static bool zend_mm_use_huge_pages = false;
204
205 /*
206 * Memory is retrieved from OS by chunks of fixed size 2MB.
207 * Inside chunk it's managed by pages of fixed size 4096B.
208 * So each chunk consists from 512 pages.
209 * The first page of each chunk is reserved for chunk header.
210 * It contains service information about all pages.
211 *
212 * free_pages - current number of free pages in this chunk
213 *
214 * free_tail - number of continuous free pages at the end of chunk
215 *
216 * free_map - bitset (a bit for each page). The bit is set if the corresponding
217 * page is allocated. Allocator for "large sizes" may easily find a
218 * free page (or a continuous number of pages) searching for zero
219 * bits.
220 *
221 * map - contains service information for each page. (32-bits for each
222 * page).
223 * usage:
224 * (2 bits)
225 * FRUN - free page,
226 * LRUN - first page of "large" allocation
227 * SRUN - first page of a bin used for "small" allocation
228 *
229 * lrun_pages:
230 * (10 bits) number of allocated pages
231 *
232 * srun_bin_num:
233 * (5 bits) bin number (e.g. 0 for sizes 0-2, 1 for 3-4,
234 * 2 for 5-8, 3 for 9-16 etc) see zend_alloc_sizes.h
235 */
236
237 struct _zend_mm_heap {
238 #if ZEND_MM_CUSTOM
239 int use_custom_heap;
240 #endif
241 #if ZEND_MM_STORAGE
242 zend_mm_storage *storage;
243 #endif
244 #if ZEND_MM_STAT
245 size_t size; /* current memory usage */
246 size_t peak; /* peak memory usage */
247 #endif
248 zend_mm_free_slot *free_slot[ZEND_MM_BINS]; /* free lists for small sizes */
249 #if ZEND_MM_STAT || ZEND_MM_LIMIT
250 size_t real_size; /* current size of allocated pages */
251 #endif
252 #if ZEND_MM_STAT
253 size_t real_peak; /* peak size of allocated pages */
254 #endif
255 #if ZEND_MM_LIMIT
256 size_t limit; /* memory limit */
257 int overflow; /* memory overflow flag */
258 #endif
259
260 zend_mm_huge_list *huge_list; /* list of huge allocated blocks */
261
262 zend_mm_chunk *main_chunk;
263 zend_mm_chunk *cached_chunks; /* list of unused chunks */
264 int chunks_count; /* number of allocated chunks */
265 int peak_chunks_count; /* peak number of allocated chunks for current request */
266 int cached_chunks_count; /* number of cached chunks */
267 double avg_chunks_count; /* average number of chunks allocated per request */
268 int last_chunks_delete_boundary; /* number of chunks after last deletion */
269 int last_chunks_delete_count; /* number of deletion over the last boundary */
270 #if ZEND_MM_CUSTOM
271 union {
272 struct {
273 void *(*_malloc)(size_t);
274 void (*_free)(void*);
275 void *(*_realloc)(void*, size_t);
276 } std;
277 struct {
278 void *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
279 void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
280 void *(*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
281 } debug;
282 } custom_heap;
283 HashTable *tracked_allocs;
284 #endif
285 };
286
287 struct _zend_mm_chunk {
288 zend_mm_heap *heap;
289 zend_mm_chunk *next;
290 zend_mm_chunk *prev;
291 uint32_t free_pages; /* number of free pages */
292 uint32_t free_tail; /* number of free pages at the end of chunk */
293 uint32_t num;
294 char reserve[64 - (sizeof(void*) * 3 + sizeof(uint32_t) * 3)];
295 zend_mm_heap heap_slot; /* used only in main chunk */
296 zend_mm_page_map free_map; /* 512 bits or 64 bytes */
297 zend_mm_page_info map[ZEND_MM_PAGES]; /* 2 KB = 512 * 4 */
298 };
299
300 struct _zend_mm_page {
301 char bytes[ZEND_MM_PAGE_SIZE];
302 };
303
304 /*
305 * bin - is one or few continuous pages (up to 8) used for allocation of
306 * a particular "small size".
307 */
308 struct _zend_mm_bin {
309 char bytes[ZEND_MM_PAGE_SIZE * 8];
310 };
311
312 struct _zend_mm_free_slot {
313 zend_mm_free_slot *next_free_slot;
314 };
315
316 struct _zend_mm_huge_list {
317 void *ptr;
318 size_t size;
319 zend_mm_huge_list *next;
320 #if ZEND_DEBUG
321 zend_mm_debug_info dbg;
322 #endif
323 };
324
325 #define ZEND_MM_PAGE_ADDR(chunk, page_num) \
326 ((void*)(((zend_mm_page*)(chunk)) + (page_num)))
327
328 #define _BIN_DATA_SIZE(num, size, elements, pages, x, y) size,
329 static const uint32_t bin_data_size[] = {
330 ZEND_MM_BINS_INFO(_BIN_DATA_SIZE, x, y)
331 };
332
333 #define _BIN_DATA_ELEMENTS(num, size, elements, pages, x, y) elements,
334 static const uint32_t bin_elements[] = {
335 ZEND_MM_BINS_INFO(_BIN_DATA_ELEMENTS, x, y)
336 };
337
338 #define _BIN_DATA_PAGES(num, size, elements, pages, x, y) pages,
339 static const uint32_t bin_pages[] = {
340 ZEND_MM_BINS_INFO(_BIN_DATA_PAGES, x, y)
341 };
342
343 #if ZEND_DEBUG
zend_debug_alloc_output(char * format,...)344 ZEND_COLD void zend_debug_alloc_output(char *format, ...)
345 {
346 char output_buf[256];
347 va_list args;
348
349 va_start(args, format);
350 vsprintf(output_buf, format, args);
351 va_end(args);
352
353 #ifdef ZEND_WIN32
354 OutputDebugString(output_buf);
355 #else
356 fprintf(stderr, "%s", output_buf);
357 #endif
358 }
359 #endif
360
zend_mm_panic(const char * message)361 static ZEND_COLD ZEND_NORETURN void zend_mm_panic(const char *message)
362 {
363 fprintf(stderr, "%s\n", message);
364 /* See http://support.microsoft.com/kb/190351 */
365 #ifdef ZEND_WIN32
366 fflush(stderr);
367 #endif
368 #if ZEND_DEBUG && defined(HAVE_KILL) && defined(HAVE_GETPID)
369 kill(getpid(), SIGSEGV);
370 #endif
371 abort();
372 }
373
zend_mm_safe_error(zend_mm_heap * heap,const char * format,size_t limit,const char * filename,uint32_t lineno,size_t size)374 static ZEND_COLD ZEND_NORETURN void zend_mm_safe_error(zend_mm_heap *heap,
375 const char *format,
376 size_t limit,
377 #if ZEND_DEBUG
378 const char *filename,
379 uint32_t lineno,
380 #endif
381 size_t size)
382 {
383
384 heap->overflow = 1;
385 zend_try {
386 zend_error_noreturn(E_ERROR,
387 format,
388 limit,
389 #if ZEND_DEBUG
390 filename,
391 lineno,
392 #endif
393 size);
394 } zend_catch {
395 } zend_end_try();
396 heap->overflow = 0;
397 zend_bailout();
398 exit(1);
399 }
400
401 #ifdef _WIN32
stderr_last_error(char * msg)402 static void stderr_last_error(char *msg)
403 {
404 DWORD err = GetLastError();
405 char *buf = php_win32_error_to_msg(err);
406
407 if (!buf[0]) {
408 fprintf(stderr, "\n%s: [0x%08lx]\n", msg, err);
409 }
410 else {
411 fprintf(stderr, "\n%s: [0x%08lx] %s\n", msg, err, buf);
412 }
413
414 php_win32_error_msg_free(buf);
415 }
416 #endif
417
418 /*****************/
419 /* OS Allocation */
420 /*****************/
421
zend_mm_munmap(void * addr,size_t size)422 static void zend_mm_munmap(void *addr, size_t size)
423 {
424 #ifdef _WIN32
425 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
426 /** ERROR_INVALID_ADDRESS is expected when addr is not range start address */
427 if (GetLastError() != ERROR_INVALID_ADDRESS) {
428 #if ZEND_MM_ERROR
429 stderr_last_error("VirtualFree() failed");
430 #endif
431 return;
432 }
433 SetLastError(0);
434
435 MEMORY_BASIC_INFORMATION mbi;
436 if (VirtualQuery(addr, &mbi, sizeof(mbi)) == 0) {
437 #if ZEND_MM_ERROR
438 stderr_last_error("VirtualQuery() failed");
439 #endif
440 return;
441 }
442 addr = mbi.AllocationBase;
443
444 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
445 #if ZEND_MM_ERROR
446 stderr_last_error("VirtualFree() failed");
447 #endif
448 }
449 }
450 #else
451 if (munmap(addr, size) != 0) {
452 #if ZEND_MM_ERROR
453 fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno));
454 #endif
455 }
456 #endif
457 }
458
459 #ifndef HAVE_MREMAP
zend_mm_mmap_fixed(void * addr,size_t size)460 static void *zend_mm_mmap_fixed(void *addr, size_t size)
461 {
462 #ifdef _WIN32
463 void *ptr = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
464
465 if (ptr == NULL) {
466 /** ERROR_INVALID_ADDRESS is expected when fixed addr range is not free */
467 if (GetLastError() != ERROR_INVALID_ADDRESS) {
468 #if ZEND_MM_ERROR
469 stderr_last_error("VirtualAlloc() fixed failed");
470 #endif
471 }
472 SetLastError(0);
473 return NULL;
474 }
475 ZEND_ASSERT(ptr == addr);
476 return ptr;
477 #else
478 int flags = MAP_PRIVATE | MAP_ANON;
479 #if defined(MAP_EXCL)
480 flags |= MAP_FIXED | MAP_EXCL;
481 #elif defined(MAP_TRYFIXED)
482 flags |= MAP_TRYFIXED;
483 #endif
484 /* MAP_FIXED leads to discarding of the old mapping, so it can't be used. */
485 void *ptr = mmap(addr, size, PROT_READ | PROT_WRITE, flags /*| MAP_POPULATE | MAP_HUGETLB*/, ZEND_MM_FD, 0);
486
487 if (ptr == MAP_FAILED) {
488 #if ZEND_MM_ERROR && !defined(MAP_EXCL) && !defined(MAP_TRYFIXED)
489 fprintf(stderr, "\nmmap() fixed failed: [%d] %s\n", errno, strerror(errno));
490 #endif
491 return NULL;
492 } else if (ptr != addr) {
493 zend_mm_munmap(ptr, size);
494 return NULL;
495 }
496 return ptr;
497 #endif
498 }
499 #endif
500
zend_mm_mmap(size_t size)501 static void *zend_mm_mmap(size_t size)
502 {
503 #ifdef _WIN32
504 void *ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
505
506 if (ptr == NULL) {
507 #if ZEND_MM_ERROR
508 stderr_last_error("VirtualAlloc() failed");
509 #endif
510 return NULL;
511 }
512 return ptr;
513 #else
514 void *ptr;
515
516 #if defined(MAP_HUGETLB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB)
517 if (zend_mm_use_huge_pages && size == ZEND_MM_CHUNK_SIZE) {
518 int fd = -1;
519 int mflags = MAP_PRIVATE | MAP_ANON;
520 #if defined(MAP_HUGETLB)
521 mflags |= MAP_HUGETLB;
522 #else
523 fd = VM_FLAGS_SUPERPAGE_SIZE_2MB;
524 #endif
525 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, mflags, fd, 0);
526 if (ptr != MAP_FAILED) {
527 zend_mmap_set_name(ptr, size, "zend_alloc");
528 return ptr;
529 }
530 }
531 #endif
532
533 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, ZEND_MM_FD, 0);
534
535 if (ptr == MAP_FAILED) {
536 #if ZEND_MM_ERROR
537 fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno));
538 #endif
539 return NULL;
540 }
541 zend_mmap_set_name(ptr, size, "zend_alloc");
542 return ptr;
543 #endif
544 }
545
546 /***********/
547 /* Bitmask */
548 /***********/
549
550 /* number of trailing set (1) bits */
zend_mm_bitset_nts(zend_mm_bitset bitset)551 static zend_always_inline int zend_mm_bitset_nts(zend_mm_bitset bitset)
552 {
553 #if (defined(__GNUC__) || __has_builtin(__builtin_ctzl)) && SIZEOF_ZEND_LONG == SIZEOF_LONG && defined(PHP_HAVE_BUILTIN_CTZL)
554 return __builtin_ctzl(~bitset);
555 #elif (defined(__GNUC__) || __has_builtin(__builtin_ctzll)) && defined(PHP_HAVE_BUILTIN_CTZLL)
556 return __builtin_ctzll(~bitset);
557 #elif defined(_WIN32)
558 unsigned long index;
559
560 #if defined(_WIN64)
561 if (!BitScanForward64(&index, ~bitset)) {
562 #else
563 if (!BitScanForward(&index, ~bitset)) {
564 #endif
565 /* undefined behavior */
566 return 32;
567 }
568
569 return (int)index;
570 #else
571 int n;
572
573 if (bitset == (zend_mm_bitset)-1) return ZEND_MM_BITSET_LEN;
574
575 n = 0;
576 #if SIZEOF_ZEND_LONG == 8
577 if (sizeof(zend_mm_bitset) == 8) {
578 if ((bitset & 0xffffffff) == 0xffffffff) {n += 32; bitset = bitset >> Z_UL(32);}
579 }
580 #endif
581 if ((bitset & 0x0000ffff) == 0x0000ffff) {n += 16; bitset = bitset >> 16;}
582 if ((bitset & 0x000000ff) == 0x000000ff) {n += 8; bitset = bitset >> 8;}
583 if ((bitset & 0x0000000f) == 0x0000000f) {n += 4; bitset = bitset >> 4;}
584 if ((bitset & 0x00000003) == 0x00000003) {n += 2; bitset = bitset >> 2;}
585 return n + (bitset & 1);
586 #endif
587 }
588
589 static zend_always_inline int zend_mm_bitset_is_set(zend_mm_bitset *bitset, int bit)
590 {
591 return ZEND_BIT_TEST(bitset, bit);
592 }
593
594 static zend_always_inline void zend_mm_bitset_set_bit(zend_mm_bitset *bitset, int bit)
595 {
596 bitset[bit / ZEND_MM_BITSET_LEN] |= (Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
597 }
598
599 static zend_always_inline void zend_mm_bitset_reset_bit(zend_mm_bitset *bitset, int bit)
600 {
601 bitset[bit / ZEND_MM_BITSET_LEN] &= ~(Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
602 }
603
604 static zend_always_inline void zend_mm_bitset_set_range(zend_mm_bitset *bitset, int start, int len)
605 {
606 if (len == 1) {
607 zend_mm_bitset_set_bit(bitset, start);
608 } else {
609 int pos = start / ZEND_MM_BITSET_LEN;
610 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
611 int bit = start & (ZEND_MM_BITSET_LEN - 1);
612 zend_mm_bitset tmp;
613
614 if (pos != end) {
615 /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
616 tmp = (zend_mm_bitset)-1 << bit;
617 bitset[pos++] |= tmp;
618 while (pos != end) {
619 /* set all bits */
620 bitset[pos++] = (zend_mm_bitset)-1;
621 }
622 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
623 /* set bits from "0" to "end" */
624 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
625 bitset[pos] |= tmp;
626 } else {
627 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
628 /* set bits from "bit" to "end" */
629 tmp = (zend_mm_bitset)-1 << bit;
630 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
631 bitset[pos] |= tmp;
632 }
633 }
634 }
635
636 static zend_always_inline void zend_mm_bitset_reset_range(zend_mm_bitset *bitset, int start, int len)
637 {
638 if (len == 1) {
639 zend_mm_bitset_reset_bit(bitset, start);
640 } else {
641 int pos = start / ZEND_MM_BITSET_LEN;
642 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
643 int bit = start & (ZEND_MM_BITSET_LEN - 1);
644 zend_mm_bitset tmp;
645
646 if (pos != end) {
647 /* reset bits from "bit" to ZEND_MM_BITSET_LEN-1 */
648 tmp = ~((Z_UL(1) << bit) - 1);
649 bitset[pos++] &= ~tmp;
650 while (pos != end) {
651 /* set all bits */
652 bitset[pos++] = 0;
653 }
654 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
655 /* reset bits from "0" to "end" */
656 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
657 bitset[pos] &= ~tmp;
658 } else {
659 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
660 /* reset bits from "bit" to "end" */
661 tmp = (zend_mm_bitset)-1 << bit;
662 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
663 bitset[pos] &= ~tmp;
664 }
665 }
666 }
667
668 static zend_always_inline int zend_mm_bitset_is_free_range(zend_mm_bitset *bitset, int start, int len)
669 {
670 if (len == 1) {
671 return !zend_mm_bitset_is_set(bitset, start);
672 } else {
673 int pos = start / ZEND_MM_BITSET_LEN;
674 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
675 int bit = start & (ZEND_MM_BITSET_LEN - 1);
676 zend_mm_bitset tmp;
677
678 if (pos != end) {
679 /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
680 tmp = (zend_mm_bitset)-1 << bit;
681 if ((bitset[pos++] & tmp) != 0) {
682 return 0;
683 }
684 while (pos != end) {
685 /* set all bits */
686 if (bitset[pos++] != 0) {
687 return 0;
688 }
689 }
690 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
691 /* set bits from "0" to "end" */
692 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
693 return (bitset[pos] & tmp) == 0;
694 } else {
695 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
696 /* set bits from "bit" to "end" */
697 tmp = (zend_mm_bitset)-1 << bit;
698 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
699 return (bitset[pos] & tmp) == 0;
700 }
701 }
702 }
703
704 /**********/
705 /* Chunks */
706 /**********/
707
708 static zend_always_inline void zend_mm_hugepage(void* ptr, size_t size)
709 {
710 #if defined(MADV_HUGEPAGE)
711 (void)madvise(ptr, size, MADV_HUGEPAGE);
712 #elif defined(HAVE_MEMCNTL)
713 struct memcntl_mha m = {.mha_cmd = MHA_MAPSIZE_VA, .mha_pagesize = ZEND_MM_CHUNK_SIZE, .mha_flags = 0};
714 (void)memcntl(ptr, size, MC_HAT_ADVISE, (char *)&m, 0, 0);
715 #elif !defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && !defined(MAP_ALIGNED_SUPER)
716 zend_error_noreturn(E_ERROR, "huge_pages: thp unsupported on this platform");
717 #endif
718 }
719
720 static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment)
721 {
722 void *ptr = zend_mm_mmap(size);
723
724 if (ptr == NULL) {
725 return NULL;
726 } else if (ZEND_MM_ALIGNED_OFFSET(ptr, alignment) == 0) {
727 if (zend_mm_use_huge_pages) {
728 zend_mm_hugepage(ptr, size);
729 }
730 #ifdef __SANITIZE_ADDRESS__
731 ASAN_UNPOISON_MEMORY_REGION(ptr, size);
732 #endif
733 return ptr;
734 } else {
735 size_t offset;
736
737 /* chunk has to be aligned */
738 zend_mm_munmap(ptr, size);
739 ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
740 #ifdef _WIN32
741 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
742 if (offset != 0) {
743 offset = alignment - offset;
744 }
745 zend_mm_munmap(ptr, size + alignment - REAL_PAGE_SIZE);
746 ptr = zend_mm_mmap_fixed((void*)((char*)ptr + offset), size);
747 if (ptr == NULL) { // fix GH-9650, fixed addr range is not free
748 ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
749 if (ptr == NULL) {
750 return NULL;
751 }
752 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
753 if (offset != 0) {
754 ptr = (void*)((char*)ptr + alignment - offset);
755 }
756 }
757 return ptr;
758 #else
759 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
760 if (offset != 0) {
761 offset = alignment - offset;
762 zend_mm_munmap(ptr, offset);
763 ptr = (char*)ptr + offset;
764 alignment -= offset;
765 }
766 if (alignment > REAL_PAGE_SIZE) {
767 zend_mm_munmap((char*)ptr + size, alignment - REAL_PAGE_SIZE);
768 }
769 if (zend_mm_use_huge_pages) {
770 zend_mm_hugepage(ptr, size);
771 }
772 # ifdef __SANITIZE_ADDRESS__
773 ASAN_UNPOISON_MEMORY_REGION(ptr, size);
774 # endif
775 #endif
776 return ptr;
777 }
778 }
779
780 static void *zend_mm_chunk_alloc(zend_mm_heap *heap, size_t size, size_t alignment)
781 {
782 #if ZEND_MM_STORAGE
783 if (UNEXPECTED(heap->storage)) {
784 void *ptr = heap->storage->handlers.chunk_alloc(heap->storage, size, alignment);
785 ZEND_ASSERT(((uintptr_t)((char*)ptr + (alignment-1)) & (alignment-1)) == (uintptr_t)ptr);
786 return ptr;
787 }
788 #endif
789 return zend_mm_chunk_alloc_int(size, alignment);
790 }
791
792 static void zend_mm_chunk_free(zend_mm_heap *heap, void *addr, size_t size)
793 {
794 #if ZEND_MM_STORAGE
795 if (UNEXPECTED(heap->storage)) {
796 heap->storage->handlers.chunk_free(heap->storage, addr, size);
797 return;
798 }
799 #endif
800 zend_mm_munmap(addr, size);
801 }
802
803 static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
804 {
805 #if ZEND_MM_STORAGE
806 if (UNEXPECTED(heap->storage)) {
807 if (heap->storage->handlers.chunk_truncate) {
808 return heap->storage->handlers.chunk_truncate(heap->storage, addr, old_size, new_size);
809 } else {
810 return 0;
811 }
812 }
813 #endif
814 #ifndef _WIN32
815 zend_mm_munmap((char*)addr + new_size, old_size - new_size);
816 return 1;
817 #else
818 return 0;
819 #endif
820 }
821
822 static int zend_mm_chunk_extend(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
823 {
824 #if ZEND_MM_STORAGE
825 if (UNEXPECTED(heap->storage)) {
826 if (heap->storage->handlers.chunk_extend) {
827 return heap->storage->handlers.chunk_extend(heap->storage, addr, old_size, new_size);
828 } else {
829 return 0;
830 }
831 }
832 #endif
833 #ifdef HAVE_MREMAP
834 /* We don't use MREMAP_MAYMOVE due to alignment requirements. */
835 void *ptr = mremap(addr, old_size, new_size, 0);
836 if (ptr == MAP_FAILED) {
837 return 0;
838 }
839 /* Sanity check: The mapping shouldn't have moved. */
840 ZEND_ASSERT(ptr == addr);
841 return 1;
842 #elif !defined(_WIN32)
843 return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL);
844 #else
845 return 0;
846 #endif
847 }
848
849 static zend_always_inline void zend_mm_chunk_init(zend_mm_heap *heap, zend_mm_chunk *chunk)
850 {
851 chunk->heap = heap;
852 chunk->next = heap->main_chunk;
853 chunk->prev = heap->main_chunk->prev;
854 chunk->prev->next = chunk;
855 chunk->next->prev = chunk;
856 /* mark first pages as allocated */
857 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
858 chunk->free_tail = ZEND_MM_FIRST_PAGE;
859 /* the younger chunks have bigger number */
860 chunk->num = chunk->prev->num + 1;
861 /* mark first pages as allocated */
862 chunk->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
863 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
864 }
865
866 /***********************/
867 /* Huge Runs (forward) */
868 /***********************/
869
870 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
871 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
872 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
873
874 #if ZEND_DEBUG
875 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
876 #else
877 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
878 #endif
879
880 /**************/
881 /* Large Runs */
882 /**************/
883
884 #if ZEND_DEBUG
885 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
886 #else
887 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
888 #endif
889 {
890 zend_mm_chunk *chunk = heap->main_chunk;
891 uint32_t page_num, len;
892 int steps = 0;
893
894 while (1) {
895 if (UNEXPECTED(chunk->free_pages < pages_count)) {
896 goto not_found;
897 #if 0
898 } else if (UNEXPECTED(chunk->free_pages + chunk->free_tail == ZEND_MM_PAGES)) {
899 if (UNEXPECTED(ZEND_MM_PAGES - chunk->free_tail < pages_count)) {
900 goto not_found;
901 } else {
902 page_num = chunk->free_tail;
903 goto found;
904 }
905 } else if (0) {
906 /* First-Fit Search */
907 int free_tail = chunk->free_tail;
908 zend_mm_bitset *bitset = chunk->free_map;
909 zend_mm_bitset tmp = *(bitset++);
910 int i = 0;
911
912 while (1) {
913 /* skip allocated blocks */
914 while (tmp == (zend_mm_bitset)-1) {
915 i += ZEND_MM_BITSET_LEN;
916 if (i == ZEND_MM_PAGES) {
917 goto not_found;
918 }
919 tmp = *(bitset++);
920 }
921 /* find first 0 bit */
922 page_num = i + zend_mm_bitset_nts(tmp);
923 /* reset bits from 0 to "bit" */
924 tmp &= tmp + 1;
925 /* skip free blocks */
926 while (tmp == 0) {
927 i += ZEND_MM_BITSET_LEN;
928 len = i - page_num;
929 if (len >= pages_count) {
930 goto found;
931 } else if (i >= free_tail) {
932 goto not_found;
933 }
934 tmp = *(bitset++);
935 }
936 /* find first 1 bit */
937 len = (i + zend_ulong_ntz(tmp)) - page_num;
938 if (len >= pages_count) {
939 goto found;
940 }
941 /* set bits from 0 to "bit" */
942 tmp |= tmp - 1;
943 }
944 #endif
945 } else {
946 /* Best-Fit Search */
947 int best = -1;
948 uint32_t best_len = ZEND_MM_PAGES;
949 uint32_t free_tail = chunk->free_tail;
950 zend_mm_bitset *bitset = chunk->free_map;
951 zend_mm_bitset tmp = *(bitset++);
952 uint32_t i = 0;
953
954 while (1) {
955 /* skip allocated blocks */
956 while (tmp == (zend_mm_bitset)-1) {
957 i += ZEND_MM_BITSET_LEN;
958 if (i == ZEND_MM_PAGES) {
959 if (best > 0) {
960 page_num = best;
961 goto found;
962 } else {
963 goto not_found;
964 }
965 }
966 tmp = *(bitset++);
967 }
968 /* find first 0 bit */
969 page_num = i + zend_mm_bitset_nts(tmp);
970 /* reset bits from 0 to "bit" */
971 tmp &= tmp + 1;
972 /* skip free blocks */
973 while (tmp == 0) {
974 i += ZEND_MM_BITSET_LEN;
975 if (i >= free_tail || i == ZEND_MM_PAGES) {
976 len = ZEND_MM_PAGES - page_num;
977 if (len >= pages_count && len < best_len) {
978 chunk->free_tail = page_num + pages_count;
979 goto found;
980 } else {
981 /* set accurate value */
982 chunk->free_tail = page_num;
983 if (best > 0) {
984 page_num = best;
985 goto found;
986 } else {
987 goto not_found;
988 }
989 }
990 }
991 tmp = *(bitset++);
992 }
993 /* find first 1 bit */
994 len = i + zend_ulong_ntz(tmp) - page_num;
995 if (len >= pages_count) {
996 if (len == pages_count) {
997 goto found;
998 } else if (len < best_len) {
999 best_len = len;
1000 best = page_num;
1001 }
1002 }
1003 /* set bits from 0 to "bit" */
1004 tmp |= tmp - 1;
1005 }
1006 }
1007
1008 not_found:
1009 if (chunk->next == heap->main_chunk) {
1010 get_chunk:
1011 if (heap->cached_chunks) {
1012 heap->cached_chunks_count--;
1013 chunk = heap->cached_chunks;
1014 heap->cached_chunks = chunk->next;
1015 } else {
1016 #if ZEND_MM_LIMIT
1017 if (UNEXPECTED(ZEND_MM_CHUNK_SIZE > heap->limit - heap->real_size)) {
1018 if (zend_mm_gc(heap)) {
1019 goto get_chunk;
1020 } else if (heap->overflow == 0) {
1021 #if ZEND_DEBUG
1022 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1023 #else
1024 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, ZEND_MM_PAGE_SIZE * pages_count);
1025 #endif
1026 return NULL;
1027 }
1028 }
1029 #endif
1030 chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1031 if (UNEXPECTED(chunk == NULL)) {
1032 /* insufficient memory */
1033 if (zend_mm_gc(heap) &&
1034 (chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE)) != NULL) {
1035 /* pass */
1036 } else {
1037 #if !ZEND_MM_LIMIT
1038 zend_mm_safe_error(heap, "Out of memory");
1039 #elif ZEND_DEBUG
1040 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1041 #else
1042 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, ZEND_MM_PAGE_SIZE * pages_count);
1043 #endif
1044 return NULL;
1045 }
1046 }
1047 #if ZEND_MM_STAT
1048 do {
1049 size_t size = heap->real_size + ZEND_MM_CHUNK_SIZE;
1050 size_t peak = MAX(heap->real_peak, size);
1051 heap->real_size = size;
1052 heap->real_peak = peak;
1053 } while (0);
1054 #elif ZEND_MM_LIMIT
1055 heap->real_size += ZEND_MM_CHUNK_SIZE;
1056
1057 #endif
1058 }
1059 heap->chunks_count++;
1060 if (heap->chunks_count > heap->peak_chunks_count) {
1061 heap->peak_chunks_count = heap->chunks_count;
1062 }
1063 zend_mm_chunk_init(heap, chunk);
1064 page_num = ZEND_MM_FIRST_PAGE;
1065 len = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1066 goto found;
1067 } else {
1068 chunk = chunk->next;
1069 steps++;
1070 }
1071 }
1072
1073 found:
1074 if (steps > 2 && pages_count < 8) {
1075 /* move chunk into the head of the linked-list */
1076 chunk->prev->next = chunk->next;
1077 chunk->next->prev = chunk->prev;
1078 chunk->next = heap->main_chunk->next;
1079 chunk->prev = heap->main_chunk;
1080 chunk->prev->next = chunk;
1081 chunk->next->prev = chunk;
1082 }
1083 /* mark run as allocated */
1084 chunk->free_pages -= pages_count;
1085 zend_mm_bitset_set_range(chunk->free_map, page_num, pages_count);
1086 chunk->map[page_num] = ZEND_MM_LRUN(pages_count);
1087 if (page_num == chunk->free_tail) {
1088 chunk->free_tail = page_num + pages_count;
1089 }
1090 return ZEND_MM_PAGE_ADDR(chunk, page_num);
1091 }
1092
1093 static zend_always_inline void *zend_mm_alloc_large_ex(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1094 {
1095 int pages_count = (int)ZEND_MM_SIZE_TO_NUM(size, ZEND_MM_PAGE_SIZE);
1096 #if ZEND_DEBUG
1097 void *ptr = zend_mm_alloc_pages(heap, pages_count, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1098 #else
1099 void *ptr = zend_mm_alloc_pages(heap, pages_count ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1100 #endif
1101 #if ZEND_MM_STAT
1102 do {
1103 size_t size = heap->size + pages_count * ZEND_MM_PAGE_SIZE;
1104 size_t peak = MAX(heap->peak, size);
1105 heap->size = size;
1106 heap->peak = peak;
1107 } while (0);
1108 #endif
1109 return ptr;
1110 }
1111
1112 #if ZEND_DEBUG
1113 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1114 {
1115 return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1116 }
1117 #else
1118 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1119 {
1120 return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1121 }
1122 #endif
1123
1124 static zend_always_inline void zend_mm_delete_chunk(zend_mm_heap *heap, zend_mm_chunk *chunk)
1125 {
1126 chunk->next->prev = chunk->prev;
1127 chunk->prev->next = chunk->next;
1128 heap->chunks_count--;
1129 if (heap->chunks_count + heap->cached_chunks_count < heap->avg_chunks_count + 0.1
1130 || (heap->chunks_count == heap->last_chunks_delete_boundary
1131 && heap->last_chunks_delete_count >= 4)) {
1132 /* delay deletion */
1133 heap->cached_chunks_count++;
1134 chunk->next = heap->cached_chunks;
1135 heap->cached_chunks = chunk;
1136 } else {
1137 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1138 heap->real_size -= ZEND_MM_CHUNK_SIZE;
1139 #endif
1140 if (!heap->cached_chunks) {
1141 if (heap->chunks_count != heap->last_chunks_delete_boundary) {
1142 heap->last_chunks_delete_boundary = heap->chunks_count;
1143 heap->last_chunks_delete_count = 0;
1144 } else {
1145 heap->last_chunks_delete_count++;
1146 }
1147 }
1148 if (!heap->cached_chunks || chunk->num > heap->cached_chunks->num) {
1149 zend_mm_chunk_free(heap, chunk, ZEND_MM_CHUNK_SIZE);
1150 } else {
1151 //TODO: select the best chunk to delete???
1152 chunk->next = heap->cached_chunks->next;
1153 zend_mm_chunk_free(heap, heap->cached_chunks, ZEND_MM_CHUNK_SIZE);
1154 heap->cached_chunks = chunk;
1155 }
1156 }
1157 }
1158
1159 static zend_always_inline void zend_mm_free_pages_ex(zend_mm_heap *heap, zend_mm_chunk *chunk, uint32_t page_num, uint32_t pages_count, int free_chunk)
1160 {
1161 chunk->free_pages += pages_count;
1162 zend_mm_bitset_reset_range(chunk->free_map, page_num, pages_count);
1163 chunk->map[page_num] = 0;
1164 if (chunk->free_tail == page_num + pages_count) {
1165 /* this setting may be not accurate */
1166 chunk->free_tail = page_num;
1167 }
1168 if (free_chunk && chunk != heap->main_chunk && chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
1169 zend_mm_delete_chunk(heap, chunk);
1170 }
1171 }
1172
1173 static zend_never_inline void zend_mm_free_pages(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1174 {
1175 zend_mm_free_pages_ex(heap, chunk, page_num, pages_count, 1);
1176 }
1177
1178 static zend_always_inline void zend_mm_free_large(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1179 {
1180 #if ZEND_MM_STAT
1181 heap->size -= pages_count * ZEND_MM_PAGE_SIZE;
1182 #endif
1183 zend_mm_free_pages(heap, chunk, page_num, pages_count);
1184 }
1185
1186 /**************/
1187 /* Small Runs */
1188 /**************/
1189
1190 /* higher set bit number (0->N/A, 1->1, 2->2, 4->3, 8->4, 127->7, 128->8 etc) */
1191 static zend_always_inline int zend_mm_small_size_to_bit(int size)
1192 {
1193 #if (defined(__GNUC__) || __has_builtin(__builtin_clz)) && defined(PHP_HAVE_BUILTIN_CLZ)
1194 return (__builtin_clz(size) ^ 0x1f) + 1;
1195 #elif defined(_WIN32)
1196 unsigned long index;
1197
1198 if (!BitScanReverse(&index, (unsigned long)size)) {
1199 /* undefined behavior */
1200 return 64;
1201 }
1202
1203 return (((31 - (int)index) ^ 0x1f) + 1);
1204 #else
1205 int n = 16;
1206 if (size <= 0x00ff) {n -= 8; size = size << 8;}
1207 if (size <= 0x0fff) {n -= 4; size = size << 4;}
1208 if (size <= 0x3fff) {n -= 2; size = size << 2;}
1209 if (size <= 0x7fff) {n -= 1;}
1210 return n;
1211 #endif
1212 }
1213
1214 #ifndef MAX
1215 # define MAX(a, b) (((a) > (b)) ? (a) : (b))
1216 #endif
1217
1218 #ifndef MIN
1219 # define MIN(a, b) (((a) < (b)) ? (a) : (b))
1220 #endif
1221
1222 static zend_always_inline int zend_mm_small_size_to_bin(size_t size)
1223 {
1224 #if 0
1225 int n;
1226 /*0, 1, 2, 3, 4, 5, 6, 7, 8, 9 10, 11, 12*/
1227 static const int f1[] = { 3, 3, 3, 3, 3, 3, 3, 4, 5, 6, 7, 8, 9};
1228 static const int f2[] = { 0, 0, 0, 0, 0, 0, 0, 4, 8, 12, 16, 20, 24};
1229
1230 if (UNEXPECTED(size <= 2)) return 0;
1231 n = zend_mm_small_size_to_bit(size - 1);
1232 return ((size-1) >> f1[n]) + f2[n];
1233 #else
1234 unsigned int t1, t2;
1235
1236 if (size <= 64) {
1237 /* we need to support size == 0 ... */
1238 return (size - !!size) >> 3;
1239 } else {
1240 t1 = size - 1;
1241 t2 = zend_mm_small_size_to_bit(t1) - 3;
1242 t1 = t1 >> t2;
1243 t2 = t2 - 3;
1244 t2 = t2 << 2;
1245 return (int)(t1 + t2);
1246 }
1247 #endif
1248 }
1249
1250 #define ZEND_MM_SMALL_SIZE_TO_BIN(size) zend_mm_small_size_to_bin(size)
1251
1252 static zend_never_inline void *zend_mm_alloc_small_slow(zend_mm_heap *heap, uint32_t bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1253 {
1254 zend_mm_chunk *chunk;
1255 int page_num;
1256 zend_mm_bin *bin;
1257 zend_mm_free_slot *p, *end;
1258
1259 #if ZEND_DEBUG
1260 bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num], bin_data_size[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1261 #else
1262 bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1263 #endif
1264 if (UNEXPECTED(bin == NULL)) {
1265 /* insufficient memory */
1266 return NULL;
1267 }
1268
1269 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(bin, ZEND_MM_CHUNK_SIZE);
1270 page_num = ZEND_MM_ALIGNED_OFFSET(bin, ZEND_MM_CHUNK_SIZE) / ZEND_MM_PAGE_SIZE;
1271 chunk->map[page_num] = ZEND_MM_SRUN(bin_num);
1272 if (bin_pages[bin_num] > 1) {
1273 uint32_t i = 1;
1274
1275 do {
1276 chunk->map[page_num+i] = ZEND_MM_NRUN(bin_num, i);
1277 i++;
1278 } while (i < bin_pages[bin_num]);
1279 }
1280
1281 /* create a linked list of elements from 1 to last */
1282 end = (zend_mm_free_slot*)((char*)bin + (bin_data_size[bin_num] * (bin_elements[bin_num] - 1)));
1283 heap->free_slot[bin_num] = p = (zend_mm_free_slot*)((char*)bin + bin_data_size[bin_num]);
1284 do {
1285 p->next_free_slot = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1286 #if ZEND_DEBUG
1287 do {
1288 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1289 dbg->size = 0;
1290 } while (0);
1291 #endif
1292 p = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1293 } while (p != end);
1294
1295 /* terminate list using NULL */
1296 p->next_free_slot = NULL;
1297 #if ZEND_DEBUG
1298 do {
1299 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1300 dbg->size = 0;
1301 } while (0);
1302 #endif
1303
1304 /* return first element */
1305 return bin;
1306 }
1307
1308 static zend_always_inline void *zend_mm_alloc_small(zend_mm_heap *heap, int bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1309 {
1310 #if ZEND_MM_STAT
1311 do {
1312 size_t size = heap->size + bin_data_size[bin_num];
1313 size_t peak = MAX(heap->peak, size);
1314 heap->size = size;
1315 heap->peak = peak;
1316 } while (0);
1317 #endif
1318
1319 if (EXPECTED(heap->free_slot[bin_num] != NULL)) {
1320 zend_mm_free_slot *p = heap->free_slot[bin_num];
1321 heap->free_slot[bin_num] = p->next_free_slot;
1322 return p;
1323 } else {
1324 return zend_mm_alloc_small_slow(heap, bin_num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1325 }
1326 }
1327
1328 static zend_always_inline void zend_mm_free_small(zend_mm_heap *heap, void *ptr, int bin_num)
1329 {
1330 zend_mm_free_slot *p;
1331
1332 #if ZEND_MM_STAT
1333 heap->size -= bin_data_size[bin_num];
1334 #endif
1335
1336 #if ZEND_DEBUG
1337 do {
1338 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1339 dbg->size = 0;
1340 } while (0);
1341 #endif
1342
1343 p = (zend_mm_free_slot*)ptr;
1344 p->next_free_slot = heap->free_slot[bin_num];
1345 heap->free_slot[bin_num] = p;
1346 }
1347
1348 /********/
1349 /* Heap */
1350 /********/
1351
1352 #if ZEND_DEBUG
1353 static zend_always_inline zend_mm_debug_info *zend_mm_get_debug_info(zend_mm_heap *heap, void *ptr)
1354 {
1355 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1356 zend_mm_chunk *chunk;
1357 int page_num;
1358 zend_mm_page_info info;
1359
1360 ZEND_MM_CHECK(page_offset != 0, "zend_mm_heap corrupted");
1361 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1362 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1363 info = chunk->map[page_num];
1364 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1365 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1366 int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1367 return (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1368 } else /* if (info & ZEND_MM_IS_LRUN) */ {
1369 int pages_count = ZEND_MM_LRUN_PAGES(info);
1370
1371 return (zend_mm_debug_info*)((char*)ptr + ZEND_MM_PAGE_SIZE * pages_count - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1372 }
1373 }
1374 #endif
1375
1376 static zend_always_inline void *zend_mm_alloc_heap(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1377 {
1378 void *ptr;
1379 #if ZEND_DEBUG
1380 size_t real_size = size;
1381 zend_mm_debug_info *dbg;
1382
1383 /* special handling for zero-size allocation */
1384 size = MAX(size, 1);
1385 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1386 if (UNEXPECTED(size < real_size)) {
1387 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", ZEND_MM_ALIGNED_SIZE(real_size), ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1388 return NULL;
1389 }
1390 #endif
1391 if (EXPECTED(size <= ZEND_MM_MAX_SMALL_SIZE)) {
1392 ptr = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1393 #if ZEND_DEBUG
1394 dbg = zend_mm_get_debug_info(heap, ptr);
1395 dbg->size = real_size;
1396 dbg->filename = __zend_filename;
1397 dbg->orig_filename = __zend_orig_filename;
1398 dbg->lineno = __zend_lineno;
1399 dbg->orig_lineno = __zend_orig_lineno;
1400 #endif
1401 return ptr;
1402 } else if (EXPECTED(size <= ZEND_MM_MAX_LARGE_SIZE)) {
1403 ptr = zend_mm_alloc_large(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1404 #if ZEND_DEBUG
1405 dbg = zend_mm_get_debug_info(heap, ptr);
1406 dbg->size = real_size;
1407 dbg->filename = __zend_filename;
1408 dbg->orig_filename = __zend_orig_filename;
1409 dbg->lineno = __zend_lineno;
1410 dbg->orig_lineno = __zend_orig_lineno;
1411 #endif
1412 return ptr;
1413 } else {
1414 #if ZEND_DEBUG
1415 size = real_size;
1416 #endif
1417 return zend_mm_alloc_huge(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1418 }
1419 }
1420
1421 static zend_always_inline void zend_mm_free_heap(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1422 {
1423 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1424
1425 if (UNEXPECTED(page_offset == 0)) {
1426 if (ptr != NULL) {
1427 zend_mm_free_huge(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1428 }
1429 } else {
1430 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1431 int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1432 zend_mm_page_info info = chunk->map[page_num];
1433
1434 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1435 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1436 zend_mm_free_small(heap, ptr, ZEND_MM_SRUN_BIN_NUM(info));
1437 } else /* if (info & ZEND_MM_IS_LRUN) */ {
1438 int pages_count = ZEND_MM_LRUN_PAGES(info);
1439
1440 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1441 zend_mm_free_large(heap, chunk, page_num, pages_count);
1442 }
1443 }
1444 }
1445
1446 static size_t zend_mm_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1447 {
1448 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1449
1450 if (UNEXPECTED(page_offset == 0)) {
1451 return zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1452 } else {
1453 zend_mm_chunk *chunk;
1454 #if 0 && ZEND_DEBUG
1455 zend_mm_debug_info *dbg = zend_mm_get_debug_info(heap, ptr);
1456 return dbg->size;
1457 #else
1458 int page_num;
1459 zend_mm_page_info info;
1460
1461 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1462 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1463 info = chunk->map[page_num];
1464 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1465 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1466 return bin_data_size[ZEND_MM_SRUN_BIN_NUM(info)];
1467 } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1468 return ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1469 }
1470 #endif
1471 }
1472 }
1473
1474 static zend_never_inline void *zend_mm_realloc_slow(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1475 {
1476 void *ret;
1477
1478 #if ZEND_MM_STAT
1479 do {
1480 size_t orig_peak = heap->peak;
1481 #endif
1482 ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1483 memcpy(ret, ptr, copy_size);
1484 zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1485 #if ZEND_MM_STAT
1486 heap->peak = MAX(orig_peak, heap->size);
1487 } while (0);
1488 #endif
1489 return ret;
1490 }
1491
1492 static zend_never_inline void *zend_mm_realloc_huge(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1493 {
1494 size_t old_size;
1495 size_t new_size;
1496 #if ZEND_DEBUG
1497 size_t real_size;
1498 #endif
1499
1500 old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1501 #if ZEND_DEBUG
1502 real_size = size;
1503 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1504 #endif
1505 if (size > ZEND_MM_MAX_LARGE_SIZE) {
1506 #if ZEND_DEBUG
1507 size = real_size;
1508 #endif
1509 #ifdef ZEND_WIN32
1510 /* On Windows we don't have ability to extend huge blocks in-place.
1511 * We allocate them with 2MB size granularity, to avoid many
1512 * reallocations when they are extended by small pieces
1513 */
1514 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE));
1515 #else
1516 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE);
1517 #endif
1518 if (new_size == old_size) {
1519 #if ZEND_DEBUG
1520 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1521 #else
1522 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1523 #endif
1524 return ptr;
1525 } else if (new_size < old_size) {
1526 /* unmup tail */
1527 if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) {
1528 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1529 heap->real_size -= old_size - new_size;
1530 #endif
1531 #if ZEND_MM_STAT
1532 heap->size -= old_size - new_size;
1533 #endif
1534 #if ZEND_DEBUG
1535 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1536 #else
1537 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1538 #endif
1539 return ptr;
1540 }
1541 } else /* if (new_size > old_size) */ {
1542 #if ZEND_MM_LIMIT
1543 if (UNEXPECTED(new_size - old_size > heap->limit - heap->real_size)) {
1544 if (zend_mm_gc(heap) && new_size - old_size <= heap->limit - heap->real_size) {
1545 /* pass */
1546 } else if (heap->overflow == 0) {
1547 #if ZEND_DEBUG
1548 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1549 #else
1550 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1551 #endif
1552 return NULL;
1553 }
1554 }
1555 #endif
1556 /* try to map tail right after this block */
1557 if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) {
1558 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1559 heap->real_size += new_size - old_size;
1560 #endif
1561 #if ZEND_MM_STAT
1562 heap->real_peak = MAX(heap->real_peak, heap->real_size);
1563 heap->size += new_size - old_size;
1564 heap->peak = MAX(heap->peak, heap->size);
1565 #endif
1566 #if ZEND_DEBUG
1567 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1568 #else
1569 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1570 #endif
1571 return ptr;
1572 }
1573 }
1574 }
1575
1576 return zend_mm_realloc_slow(heap, ptr, size, MIN(old_size, copy_size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1577 }
1578
1579 static zend_always_inline void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, bool use_copy_size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1580 {
1581 size_t page_offset;
1582 size_t old_size;
1583 size_t new_size;
1584 void *ret;
1585 #if ZEND_DEBUG
1586 zend_mm_debug_info *dbg;
1587 #endif
1588
1589 page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1590 if (UNEXPECTED(page_offset == 0)) {
1591 if (EXPECTED(ptr == NULL)) {
1592 return _zend_mm_alloc(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1593 } else {
1594 return zend_mm_realloc_huge(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1595 }
1596 } else {
1597 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1598 int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1599 zend_mm_page_info info = chunk->map[page_num];
1600 #if ZEND_DEBUG
1601 size_t real_size = size;
1602
1603 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1604 #endif
1605
1606 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1607 if (info & ZEND_MM_IS_SRUN) {
1608 int old_bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1609
1610 do {
1611 old_size = bin_data_size[old_bin_num];
1612
1613 /* Check if requested size fits into current bin */
1614 if (size <= old_size) {
1615 /* Check if truncation is necessary */
1616 if (old_bin_num > 0 && size < bin_data_size[old_bin_num - 1]) {
1617 /* truncation */
1618 ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1619 copy_size = use_copy_size ? MIN(size, copy_size) : size;
1620 memcpy(ret, ptr, copy_size);
1621 zend_mm_free_small(heap, ptr, old_bin_num);
1622 } else {
1623 /* reallocation in-place */
1624 ret = ptr;
1625 }
1626 } else if (size <= ZEND_MM_MAX_SMALL_SIZE) {
1627 /* small extension */
1628
1629 #if ZEND_MM_STAT
1630 do {
1631 size_t orig_peak = heap->peak;
1632 #endif
1633 ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1634 copy_size = use_copy_size ? MIN(old_size, copy_size) : old_size;
1635 memcpy(ret, ptr, copy_size);
1636 zend_mm_free_small(heap, ptr, old_bin_num);
1637 #if ZEND_MM_STAT
1638 heap->peak = MAX(orig_peak, heap->size);
1639 } while (0);
1640 #endif
1641 } else {
1642 /* slow reallocation */
1643 break;
1644 }
1645
1646 #if ZEND_DEBUG
1647 dbg = zend_mm_get_debug_info(heap, ret);
1648 dbg->size = real_size;
1649 dbg->filename = __zend_filename;
1650 dbg->orig_filename = __zend_orig_filename;
1651 dbg->lineno = __zend_lineno;
1652 dbg->orig_lineno = __zend_orig_lineno;
1653 #endif
1654 return ret;
1655 } while (0);
1656
1657 } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1658 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1659 old_size = ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1660 if (size > ZEND_MM_MAX_SMALL_SIZE && size <= ZEND_MM_MAX_LARGE_SIZE) {
1661 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE);
1662 if (new_size == old_size) {
1663 #if ZEND_DEBUG
1664 dbg = zend_mm_get_debug_info(heap, ptr);
1665 dbg->size = real_size;
1666 dbg->filename = __zend_filename;
1667 dbg->orig_filename = __zend_orig_filename;
1668 dbg->lineno = __zend_lineno;
1669 dbg->orig_lineno = __zend_orig_lineno;
1670 #endif
1671 return ptr;
1672 } else if (new_size < old_size) {
1673 /* free tail pages */
1674 int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1675 int rest_pages_count = (int)((old_size - new_size) / ZEND_MM_PAGE_SIZE);
1676
1677 #if ZEND_MM_STAT
1678 heap->size -= rest_pages_count * ZEND_MM_PAGE_SIZE;
1679 #endif
1680 chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1681 chunk->free_pages += rest_pages_count;
1682 zend_mm_bitset_reset_range(chunk->free_map, page_num + new_pages_count, rest_pages_count);
1683 #if ZEND_DEBUG
1684 dbg = zend_mm_get_debug_info(heap, ptr);
1685 dbg->size = real_size;
1686 dbg->filename = __zend_filename;
1687 dbg->orig_filename = __zend_orig_filename;
1688 dbg->lineno = __zend_lineno;
1689 dbg->orig_lineno = __zend_orig_lineno;
1690 #endif
1691 return ptr;
1692 } else /* if (new_size > old_size) */ {
1693 int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1694 int old_pages_count = (int)(old_size / ZEND_MM_PAGE_SIZE);
1695
1696 /* try to allocate tail pages after this block */
1697 if (page_num + new_pages_count <= ZEND_MM_PAGES &&
1698 zend_mm_bitset_is_free_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count)) {
1699 #if ZEND_MM_STAT
1700 do {
1701 size_t size = heap->size + (new_size - old_size);
1702 size_t peak = MAX(heap->peak, size);
1703 heap->size = size;
1704 heap->peak = peak;
1705 } while (0);
1706 #endif
1707 chunk->free_pages -= new_pages_count - old_pages_count;
1708 zend_mm_bitset_set_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count);
1709 chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1710 #if ZEND_DEBUG
1711 dbg = zend_mm_get_debug_info(heap, ptr);
1712 dbg->size = real_size;
1713 dbg->filename = __zend_filename;
1714 dbg->orig_filename = __zend_orig_filename;
1715 dbg->lineno = __zend_lineno;
1716 dbg->orig_lineno = __zend_orig_lineno;
1717 #endif
1718 return ptr;
1719 }
1720 }
1721 }
1722 }
1723 #if ZEND_DEBUG
1724 size = real_size;
1725 #endif
1726 }
1727
1728 copy_size = MIN(old_size, copy_size);
1729 return zend_mm_realloc_slow(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1730 }
1731
1732 /*********************/
1733 /* Huge Runs (again) */
1734 /*********************/
1735
1736 #if ZEND_DEBUG
1737 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1738 #else
1739 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1740 #endif
1741 {
1742 zend_mm_huge_list *list = (zend_mm_huge_list*)zend_mm_alloc_heap(heap, sizeof(zend_mm_huge_list) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1743 list->ptr = ptr;
1744 list->size = size;
1745 list->next = heap->huge_list;
1746 #if ZEND_DEBUG
1747 list->dbg.size = dbg_size;
1748 list->dbg.filename = __zend_filename;
1749 list->dbg.orig_filename = __zend_orig_filename;
1750 list->dbg.lineno = __zend_lineno;
1751 list->dbg.orig_lineno = __zend_orig_lineno;
1752 #endif
1753 heap->huge_list = list;
1754 }
1755
1756 static size_t zend_mm_del_huge_block(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1757 {
1758 zend_mm_huge_list *prev = NULL;
1759 zend_mm_huge_list *list = heap->huge_list;
1760 while (list != NULL) {
1761 if (list->ptr == ptr) {
1762 size_t size;
1763
1764 if (prev) {
1765 prev->next = list->next;
1766 } else {
1767 heap->huge_list = list->next;
1768 }
1769 size = list->size;
1770 zend_mm_free_heap(heap, list ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1771 return size;
1772 }
1773 prev = list;
1774 list = list->next;
1775 }
1776 ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1777 return 0;
1778 }
1779
1780 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1781 {
1782 zend_mm_huge_list *list = heap->huge_list;
1783 while (list != NULL) {
1784 if (list->ptr == ptr) {
1785 return list->size;
1786 }
1787 list = list->next;
1788 }
1789 ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1790 return 0;
1791 }
1792
1793 #if ZEND_DEBUG
1794 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1795 #else
1796 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1797 #endif
1798 {
1799 zend_mm_huge_list *list = heap->huge_list;
1800 while (list != NULL) {
1801 if (list->ptr == ptr) {
1802 list->size = size;
1803 #if ZEND_DEBUG
1804 list->dbg.size = dbg_size;
1805 list->dbg.filename = __zend_filename;
1806 list->dbg.orig_filename = __zend_orig_filename;
1807 list->dbg.lineno = __zend_lineno;
1808 list->dbg.orig_lineno = __zend_orig_lineno;
1809 #endif
1810 return;
1811 }
1812 list = list->next;
1813 }
1814 }
1815
1816 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1817 {
1818 #ifdef ZEND_WIN32
1819 /* On Windows we don't have ability to extend huge blocks in-place.
1820 * We allocate them with 2MB size granularity, to avoid many
1821 * reallocations when they are extended by small pieces
1822 */
1823 size_t alignment = MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE);
1824 #else
1825 size_t alignment = REAL_PAGE_SIZE;
1826 #endif
1827 size_t new_size = ZEND_MM_ALIGNED_SIZE_EX(size, alignment);
1828 void *ptr;
1829
1830 if (UNEXPECTED(new_size < size)) {
1831 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", size, alignment);
1832 }
1833
1834 #if ZEND_MM_LIMIT
1835 if (UNEXPECTED(new_size > heap->limit - heap->real_size)) {
1836 if (zend_mm_gc(heap) && new_size <= heap->limit - heap->real_size) {
1837 /* pass */
1838 } else if (heap->overflow == 0) {
1839 #if ZEND_DEBUG
1840 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1841 #else
1842 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1843 #endif
1844 return NULL;
1845 }
1846 }
1847 #endif
1848 ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE);
1849 if (UNEXPECTED(ptr == NULL)) {
1850 /* insufficient memory */
1851 if (zend_mm_gc(heap) &&
1852 (ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE)) != NULL) {
1853 /* pass */
1854 } else {
1855 #if !ZEND_MM_LIMIT
1856 zend_mm_safe_error(heap, "Out of memory");
1857 #elif ZEND_DEBUG
1858 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1859 #else
1860 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, size);
1861 #endif
1862 return NULL;
1863 }
1864 }
1865 #if ZEND_DEBUG
1866 zend_mm_add_huge_block(heap, ptr, new_size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1867 #else
1868 zend_mm_add_huge_block(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1869 #endif
1870 #if ZEND_MM_STAT
1871 do {
1872 size_t size = heap->real_size + new_size;
1873 size_t peak = MAX(heap->real_peak, size);
1874 heap->real_size = size;
1875 heap->real_peak = peak;
1876 } while (0);
1877 do {
1878 size_t size = heap->size + new_size;
1879 size_t peak = MAX(heap->peak, size);
1880 heap->size = size;
1881 heap->peak = peak;
1882 } while (0);
1883 #elif ZEND_MM_LIMIT
1884 heap->real_size += new_size;
1885 #endif
1886 return ptr;
1887 }
1888
1889 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1890 {
1891 size_t size;
1892
1893 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE) == 0, "zend_mm_heap corrupted");
1894 size = zend_mm_del_huge_block(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1895 zend_mm_chunk_free(heap, ptr, size);
1896 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1897 heap->real_size -= size;
1898 #endif
1899 #if ZEND_MM_STAT
1900 heap->size -= size;
1901 #endif
1902 }
1903
1904 /******************/
1905 /* Initialization */
1906 /******************/
1907
1908 static zend_mm_heap *zend_mm_init(void)
1909 {
1910 zend_mm_chunk *chunk = (zend_mm_chunk*)zend_mm_chunk_alloc_int(ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1911 zend_mm_heap *heap;
1912
1913 if (UNEXPECTED(chunk == NULL)) {
1914 #if ZEND_MM_ERROR
1915 fprintf(stderr, "Can't initialize heap\n");
1916 #endif
1917 return NULL;
1918 }
1919 heap = &chunk->heap_slot;
1920 chunk->heap = heap;
1921 chunk->next = chunk;
1922 chunk->prev = chunk;
1923 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1924 chunk->free_tail = ZEND_MM_FIRST_PAGE;
1925 chunk->num = 0;
1926 chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
1927 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
1928 heap->main_chunk = chunk;
1929 heap->cached_chunks = NULL;
1930 heap->chunks_count = 1;
1931 heap->peak_chunks_count = 1;
1932 heap->cached_chunks_count = 0;
1933 heap->avg_chunks_count = 1.0;
1934 heap->last_chunks_delete_boundary = 0;
1935 heap->last_chunks_delete_count = 0;
1936 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1937 heap->real_size = ZEND_MM_CHUNK_SIZE;
1938 #endif
1939 #if ZEND_MM_STAT
1940 heap->real_peak = ZEND_MM_CHUNK_SIZE;
1941 heap->size = 0;
1942 heap->peak = 0;
1943 #endif
1944 #if ZEND_MM_LIMIT
1945 heap->limit = (size_t)Z_L(-1) >> 1;
1946 heap->overflow = 0;
1947 #endif
1948 #if ZEND_MM_CUSTOM
1949 heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
1950 #endif
1951 #if ZEND_MM_STORAGE
1952 heap->storage = NULL;
1953 #endif
1954 heap->huge_list = NULL;
1955 return heap;
1956 }
1957
1958 ZEND_API size_t zend_mm_gc(zend_mm_heap *heap)
1959 {
1960 zend_mm_free_slot *p, **q;
1961 zend_mm_chunk *chunk;
1962 size_t page_offset;
1963 int page_num;
1964 zend_mm_page_info info;
1965 uint32_t i, free_counter;
1966 bool has_free_pages;
1967 size_t collected = 0;
1968
1969 #if ZEND_MM_CUSTOM
1970 if (heap->use_custom_heap) {
1971 return 0;
1972 }
1973 #endif
1974
1975 for (i = 0; i < ZEND_MM_BINS; i++) {
1976 has_free_pages = false;
1977 p = heap->free_slot[i];
1978 while (p != NULL) {
1979 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
1980 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1981 page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
1982 ZEND_ASSERT(page_offset != 0);
1983 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1984 info = chunk->map[page_num];
1985 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1986 if (info & ZEND_MM_IS_LRUN) {
1987 page_num -= ZEND_MM_NRUN_OFFSET(info);
1988 info = chunk->map[page_num];
1989 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1990 ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
1991 }
1992 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
1993 free_counter = ZEND_MM_SRUN_FREE_COUNTER(info) + 1;
1994 if (free_counter == bin_elements[i]) {
1995 has_free_pages = true;
1996 }
1997 chunk->map[page_num] = ZEND_MM_SRUN_EX(i, free_counter);
1998 p = p->next_free_slot;
1999 }
2000
2001 if (!has_free_pages) {
2002 continue;
2003 }
2004
2005 q = &heap->free_slot[i];
2006 p = *q;
2007 while (p != NULL) {
2008 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
2009 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
2010 page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
2011 ZEND_ASSERT(page_offset != 0);
2012 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
2013 info = chunk->map[page_num];
2014 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
2015 if (info & ZEND_MM_IS_LRUN) {
2016 page_num -= ZEND_MM_NRUN_OFFSET(info);
2017 info = chunk->map[page_num];
2018 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
2019 ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
2020 }
2021 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
2022 if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[i]) {
2023 /* remove from cache */
2024 p = p->next_free_slot;
2025 *q = p;
2026 } else {
2027 q = &p->next_free_slot;
2028 p = *q;
2029 }
2030 }
2031 }
2032
2033 chunk = heap->main_chunk;
2034 do {
2035 i = ZEND_MM_FIRST_PAGE;
2036 while (i < chunk->free_tail) {
2037 if (zend_mm_bitset_is_set(chunk->free_map, i)) {
2038 info = chunk->map[i];
2039 if (info & ZEND_MM_IS_SRUN) {
2040 int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
2041 int pages_count = bin_pages[bin_num];
2042
2043 if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[bin_num]) {
2044 /* all elements are free */
2045 zend_mm_free_pages_ex(heap, chunk, i, pages_count, 0);
2046 collected += pages_count;
2047 } else {
2048 /* reset counter */
2049 chunk->map[i] = ZEND_MM_SRUN(bin_num);
2050 }
2051 i += bin_pages[bin_num];
2052 } else /* if (info & ZEND_MM_IS_LRUN) */ {
2053 i += ZEND_MM_LRUN_PAGES(info);
2054 }
2055 } else {
2056 i++;
2057 }
2058 }
2059 if (chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
2060 zend_mm_chunk *next_chunk = chunk->next;
2061
2062 zend_mm_delete_chunk(heap, chunk);
2063 chunk = next_chunk;
2064 } else {
2065 chunk = chunk->next;
2066 }
2067 } while (chunk != heap->main_chunk);
2068
2069 return collected * ZEND_MM_PAGE_SIZE;
2070 }
2071
2072 #if ZEND_DEBUG
2073 /******************/
2074 /* Leak detection */
2075 /******************/
2076
2077 static zend_long zend_mm_find_leaks_small(zend_mm_chunk *p, uint32_t i, uint32_t j, zend_leak_info *leak)
2078 {
2079 bool empty = true;
2080 zend_long count = 0;
2081 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2082 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * (j + 1) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2083
2084 while (j < bin_elements[bin_num]) {
2085 if (dbg->size != 0) {
2086 if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2087 count++;
2088 dbg->size = 0;
2089 dbg->filename = NULL;
2090 dbg->lineno = 0;
2091 } else {
2092 empty = false;
2093 }
2094 }
2095 j++;
2096 dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2097 }
2098 if (empty) {
2099 zend_mm_bitset_reset_range(p->free_map, i, bin_pages[bin_num]);
2100 }
2101 return count;
2102 }
2103
2104 static zend_long zend_mm_find_leaks(zend_mm_heap *heap, zend_mm_chunk *p, uint32_t i, zend_leak_info *leak)
2105 {
2106 zend_long count = 0;
2107
2108 do {
2109 while (i < p->free_tail) {
2110 if (zend_mm_bitset_is_set(p->free_map, i)) {
2111 if (p->map[i] & ZEND_MM_IS_SRUN) {
2112 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2113 count += zend_mm_find_leaks_small(p, i, 0, leak);
2114 i += bin_pages[bin_num];
2115 } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2116 int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2117 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2118
2119 if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2120 count++;
2121 }
2122 zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2123 i += pages_count;
2124 }
2125 } else {
2126 i++;
2127 }
2128 }
2129 p = p->next;
2130 i = ZEND_MM_FIRST_PAGE;
2131 } while (p != heap->main_chunk);
2132 return count;
2133 }
2134
2135 static zend_long zend_mm_find_leaks_huge(zend_mm_heap *heap, zend_mm_huge_list *list)
2136 {
2137 zend_long count = 0;
2138 zend_mm_huge_list *prev = list;
2139 zend_mm_huge_list *p = list->next;
2140
2141 while (p) {
2142 if (p->dbg.filename == list->dbg.filename && p->dbg.lineno == list->dbg.lineno) {
2143 prev->next = p->next;
2144 zend_mm_chunk_free(heap, p->ptr, p->size);
2145 zend_mm_free_heap(heap, p, NULL, 0, NULL, 0);
2146 count++;
2147 } else {
2148 prev = p;
2149 }
2150 p = prev->next;
2151 }
2152
2153 return count;
2154 }
2155
2156 static void zend_mm_check_leaks(zend_mm_heap *heap)
2157 {
2158 zend_mm_huge_list *list;
2159 zend_mm_chunk *p;
2160 zend_leak_info leak;
2161 zend_long repeated = 0;
2162 uint32_t total = 0;
2163 uint32_t i, j;
2164
2165 /* find leaked huge blocks and free them */
2166 list = heap->huge_list;
2167 while (list) {
2168 zend_mm_huge_list *q = list;
2169
2170 leak.addr = list->ptr;
2171 leak.size = list->dbg.size;
2172 leak.filename = list->dbg.filename;
2173 leak.orig_filename = list->dbg.orig_filename;
2174 leak.lineno = list->dbg.lineno;
2175 leak.orig_lineno = list->dbg.orig_lineno;
2176
2177 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2178 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2179 repeated = zend_mm_find_leaks_huge(heap, list);
2180 total += 1 + repeated;
2181 if (repeated) {
2182 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated);
2183 }
2184
2185 heap->huge_list = list = list->next;
2186 zend_mm_chunk_free(heap, q->ptr, q->size);
2187 zend_mm_free_heap(heap, q, NULL, 0, NULL, 0);
2188 }
2189
2190 /* for each chunk */
2191 p = heap->main_chunk;
2192 do {
2193 i = ZEND_MM_FIRST_PAGE;
2194 while (i < p->free_tail) {
2195 if (zend_mm_bitset_is_set(p->free_map, i)) {
2196 if (p->map[i] & ZEND_MM_IS_SRUN) {
2197 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2198 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2199
2200 j = 0;
2201 while (j < bin_elements[bin_num]) {
2202 if (dbg->size != 0) {
2203 leak.addr = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * j);
2204 leak.size = dbg->size;
2205 leak.filename = dbg->filename;
2206 leak.orig_filename = dbg->orig_filename;
2207 leak.lineno = dbg->lineno;
2208 leak.orig_lineno = dbg->orig_lineno;
2209
2210 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2211 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2212
2213 dbg->size = 0;
2214 dbg->filename = NULL;
2215 dbg->lineno = 0;
2216
2217 repeated = zend_mm_find_leaks_small(p, i, j + 1, &leak) +
2218 zend_mm_find_leaks(heap, p, i + bin_pages[bin_num], &leak);
2219 total += 1 + repeated;
2220 if (repeated) {
2221 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated);
2222 }
2223 }
2224 dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2225 j++;
2226 }
2227 i += bin_pages[bin_num];
2228 } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2229 int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2230 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2231
2232 leak.addr = (void*)((char*)p + ZEND_MM_PAGE_SIZE * i);
2233 leak.size = dbg->size;
2234 leak.filename = dbg->filename;
2235 leak.orig_filename = dbg->orig_filename;
2236 leak.lineno = dbg->lineno;
2237 leak.orig_lineno = dbg->orig_lineno;
2238
2239 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2240 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2241
2242 zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2243
2244 repeated = zend_mm_find_leaks(heap, p, i + pages_count, &leak);
2245 total += 1 + repeated;
2246 if (repeated) {
2247 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated);
2248 }
2249 i += pages_count;
2250 }
2251 } else {
2252 i++;
2253 }
2254 }
2255 p = p->next;
2256 } while (p != heap->main_chunk);
2257 if (total) {
2258 zend_message_dispatcher(ZMSG_MEMORY_LEAKS_GRAND_TOTAL, &total);
2259 }
2260 }
2261 #endif
2262
2263 #if ZEND_MM_CUSTOM
2264 static void *tracked_malloc(size_t size);
2265 static void tracked_free_all(void);
2266 #endif
2267
2268 void zend_mm_shutdown(zend_mm_heap *heap, bool full, bool silent)
2269 {
2270 zend_mm_chunk *p;
2271 zend_mm_huge_list *list;
2272
2273 #if ZEND_MM_CUSTOM
2274 if (heap->use_custom_heap) {
2275 if (heap->custom_heap.std._malloc == tracked_malloc) {
2276 if (silent) {
2277 tracked_free_all();
2278 }
2279 zend_hash_clean(heap->tracked_allocs);
2280 if (full) {
2281 zend_hash_destroy(heap->tracked_allocs);
2282 free(heap->tracked_allocs);
2283 /* Make sure the heap free below does not use tracked_free(). */
2284 heap->custom_heap.std._free = free;
2285 }
2286 heap->size = 0;
2287 }
2288
2289 if (full) {
2290 if (ZEND_DEBUG && heap->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2291 heap->custom_heap.debug._free(heap ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC);
2292 } else {
2293 heap->custom_heap.std._free(heap);
2294 }
2295 }
2296 return;
2297 }
2298 #endif
2299
2300 #if ZEND_DEBUG
2301 if (!silent) {
2302 char *tmp = getenv("ZEND_ALLOC_PRINT_LEAKS");
2303 if (!tmp || ZEND_ATOL(tmp)) {
2304 zend_mm_check_leaks(heap);
2305 }
2306 }
2307 #endif
2308
2309 /* free huge blocks */
2310 list = heap->huge_list;
2311 heap->huge_list = NULL;
2312 while (list) {
2313 zend_mm_huge_list *q = list;
2314 list = list->next;
2315 zend_mm_chunk_free(heap, q->ptr, q->size);
2316 }
2317
2318 /* move all chunks except of the first one into the cache */
2319 p = heap->main_chunk->next;
2320 while (p != heap->main_chunk) {
2321 zend_mm_chunk *q = p->next;
2322 p->next = heap->cached_chunks;
2323 heap->cached_chunks = p;
2324 p = q;
2325 heap->chunks_count--;
2326 heap->cached_chunks_count++;
2327 }
2328
2329 if (full) {
2330 /* free all cached chunks */
2331 while (heap->cached_chunks) {
2332 p = heap->cached_chunks;
2333 heap->cached_chunks = p->next;
2334 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2335 }
2336 /* free the first chunk */
2337 zend_mm_chunk_free(heap, heap->main_chunk, ZEND_MM_CHUNK_SIZE);
2338 } else {
2339 /* free some cached chunks to keep average count */
2340 heap->avg_chunks_count = (heap->avg_chunks_count + (double)heap->peak_chunks_count) / 2.0;
2341 while ((double)heap->cached_chunks_count + 0.9 > heap->avg_chunks_count &&
2342 heap->cached_chunks) {
2343 p = heap->cached_chunks;
2344 heap->cached_chunks = p->next;
2345 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2346 heap->cached_chunks_count--;
2347 }
2348 /* clear cached chunks */
2349 p = heap->cached_chunks;
2350 while (p != NULL) {
2351 zend_mm_chunk *q = p->next;
2352 memset(p, 0, sizeof(zend_mm_chunk));
2353 p->next = q;
2354 p = q;
2355 }
2356
2357 /* reinitialize the first chunk and heap */
2358 p = heap->main_chunk;
2359 p->heap = &p->heap_slot;
2360 p->next = p;
2361 p->prev = p;
2362 p->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
2363 p->free_tail = ZEND_MM_FIRST_PAGE;
2364 p->num = 0;
2365
2366 #if ZEND_MM_STAT
2367 heap->size = heap->peak = 0;
2368 #endif
2369 memset(heap->free_slot, 0, sizeof(heap->free_slot));
2370 #if ZEND_MM_STAT || ZEND_MM_LIMIT
2371 heap->real_size = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE;
2372 #endif
2373 #if ZEND_MM_STAT
2374 heap->real_peak = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE;
2375 #endif
2376 heap->chunks_count = 1;
2377 heap->peak_chunks_count = 1;
2378 heap->last_chunks_delete_boundary = 0;
2379 heap->last_chunks_delete_count = 0;
2380
2381 memset(p->free_map, 0, sizeof(p->free_map) + sizeof(p->map));
2382 p->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
2383 p->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
2384 }
2385 }
2386
2387 /**************/
2388 /* PUBLIC API */
2389 /**************/
2390
2391 ZEND_API void* ZEND_FASTCALL _zend_mm_alloc(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2392 {
2393 return zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2394 }
2395
2396 ZEND_API void ZEND_FASTCALL _zend_mm_free(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2397 {
2398 zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2399 }
2400
2401 void* ZEND_FASTCALL _zend_mm_realloc(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2402 {
2403 return zend_mm_realloc_heap(heap, ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2404 }
2405
2406 void* ZEND_FASTCALL _zend_mm_realloc2(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2407 {
2408 return zend_mm_realloc_heap(heap, ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2409 }
2410
2411 ZEND_API size_t ZEND_FASTCALL _zend_mm_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2412 {
2413 #if ZEND_MM_CUSTOM
2414 if (UNEXPECTED(heap->use_custom_heap)) {
2415 if (heap->custom_heap.std._malloc == tracked_malloc) {
2416 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2417 zval *size_zv = zend_hash_index_find(heap->tracked_allocs, h);
2418 if (size_zv) {
2419 return Z_LVAL_P(size_zv);
2420 }
2421 }
2422 return 0;
2423 }
2424 #endif
2425 return zend_mm_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2426 }
2427
2428 /**********************/
2429 /* Allocation Manager */
2430 /**********************/
2431
2432 typedef struct _zend_alloc_globals {
2433 zend_mm_heap *mm_heap;
2434 } zend_alloc_globals;
2435
2436 #ifdef ZTS
2437 static int alloc_globals_id;
2438 static size_t alloc_globals_offset;
2439 # define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v)
2440 #else
2441 # define AG(v) (alloc_globals.v)
2442 static zend_alloc_globals alloc_globals;
2443 #endif
2444
2445 ZEND_API bool is_zend_mm(void)
2446 {
2447 #if ZEND_MM_CUSTOM
2448 return !AG(mm_heap)->use_custom_heap;
2449 #else
2450 return 1;
2451 #endif
2452 }
2453
2454 ZEND_API bool is_zend_ptr(const void *ptr)
2455 {
2456 #if ZEND_MM_CUSTOM
2457 if (AG(mm_heap)->use_custom_heap) {
2458 if (AG(mm_heap)->custom_heap.std._malloc == tracked_malloc) {
2459 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2460 zval *size_zv = zend_hash_index_find(AG(mm_heap)->tracked_allocs, h);
2461 if (size_zv) {
2462 return 1;
2463 }
2464 }
2465 return 0;
2466 }
2467 #endif
2468
2469 if (AG(mm_heap)->main_chunk) {
2470 zend_mm_chunk *chunk = AG(mm_heap)->main_chunk;
2471
2472 do {
2473 if (ptr >= (void*)chunk
2474 && ptr < (void*)((char*)chunk + ZEND_MM_CHUNK_SIZE)) {
2475 return 1;
2476 }
2477 chunk = chunk->next;
2478 } while (chunk != AG(mm_heap)->main_chunk);
2479 }
2480
2481 if (AG(mm_heap)->huge_list) {
2482 zend_mm_huge_list *block = AG(mm_heap)->huge_list;
2483
2484 do {
2485 if (ptr >= (void*)block
2486 && ptr < (void*)((char*)block + block->size)) {
2487 return 1;
2488 }
2489 block = block->next;
2490 } while (block != AG(mm_heap)->huge_list);
2491 }
2492 return 0;
2493 }
2494
2495 #if ZEND_MM_CUSTOM
2496
2497 static ZEND_COLD void* ZEND_FASTCALL _malloc_custom(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2498 {
2499 if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2500 return AG(mm_heap)->custom_heap.debug._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2501 } else {
2502 return AG(mm_heap)->custom_heap.std._malloc(size);
2503 }
2504 }
2505
2506 static ZEND_COLD void ZEND_FASTCALL _efree_custom(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2507 {
2508 if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2509 AG(mm_heap)->custom_heap.debug._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2510 } else {
2511 AG(mm_heap)->custom_heap.std._free(ptr);
2512 }
2513 }
2514
2515 static ZEND_COLD void* ZEND_FASTCALL _realloc_custom(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2516 {
2517 if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2518 return AG(mm_heap)->custom_heap.debug._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2519 } else {
2520 return AG(mm_heap)->custom_heap.std._realloc(ptr, size);
2521 }
2522 }
2523 #endif
2524
2525 #if !ZEND_DEBUG && defined(HAVE_BUILTIN_CONSTANT_P)
2526 #undef _emalloc
2527
2528 #if ZEND_MM_CUSTOM
2529 # define ZEND_MM_CUSTOM_ALLOCATOR(size) do { \
2530 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2531 return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2532 } \
2533 } while (0)
2534 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr) do { \
2535 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2536 _efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2537 return; \
2538 } \
2539 } while (0)
2540 #else
2541 # define ZEND_MM_CUSTOM_ALLOCATOR(size)
2542 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr)
2543 #endif
2544
2545 # define _ZEND_BIN_ALLOCATOR(_num, _size, _elements, _pages, x, y) \
2546 ZEND_API void* ZEND_FASTCALL _emalloc_ ## _size(void) { \
2547 ZEND_MM_CUSTOM_ALLOCATOR(_size); \
2548 return zend_mm_alloc_small(AG(mm_heap), _num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2549 }
2550
2551 ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR, x, y)
2552
2553 ZEND_API void* ZEND_FASTCALL _emalloc_large(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2554 {
2555 ZEND_MM_CUSTOM_ALLOCATOR(size);
2556 return zend_mm_alloc_large_ex(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2557 }
2558
2559 ZEND_API void* ZEND_FASTCALL _emalloc_huge(size_t size)
2560 {
2561 ZEND_MM_CUSTOM_ALLOCATOR(size);
2562 return zend_mm_alloc_huge(AG(mm_heap), size);
2563 }
2564
2565 #if ZEND_DEBUG
2566 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2567 ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2568 ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2569 { \
2570 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); \
2571 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2572 int page_num = page_offset / ZEND_MM_PAGE_SIZE; \
2573 ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2574 ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_SRUN); \
2575 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(chunk->map[page_num]) == _num); \
2576 zend_mm_free_small(AG(mm_heap), ptr, _num); \
2577 } \
2578 }
2579 #else
2580 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2581 ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2582 ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2583 { \
2584 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2585 ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2586 zend_mm_free_small(AG(mm_heap), ptr, _num); \
2587 } \
2588 }
2589 #endif
2590
2591 ZEND_MM_BINS_INFO(_ZEND_BIN_FREE, x, y)
2592
2593 ZEND_API void ZEND_FASTCALL _efree_large(void *ptr, size_t size)
2594 {
2595 ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2596 {
2597 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
2598 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
2599 int page_num = page_offset / ZEND_MM_PAGE_SIZE;
2600 uint32_t pages_count = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE) / ZEND_MM_PAGE_SIZE;
2601
2602 ZEND_MM_CHECK(chunk->heap == AG(mm_heap) && ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
2603 ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_LRUN);
2604 ZEND_ASSERT(ZEND_MM_LRUN_PAGES(chunk->map[page_num]) == pages_count);
2605 zend_mm_free_large(AG(mm_heap), chunk, page_num, pages_count);
2606 }
2607 }
2608
2609 ZEND_API void ZEND_FASTCALL _efree_huge(void *ptr, size_t size)
2610 {
2611
2612 ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2613 zend_mm_free_huge(AG(mm_heap), ptr);
2614 }
2615 #endif
2616
2617 ZEND_API void* ZEND_FASTCALL _emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2618 {
2619 #if ZEND_MM_CUSTOM
2620 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2621 return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2622 }
2623 #endif
2624 return zend_mm_alloc_heap(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2625 }
2626
2627 ZEND_API void ZEND_FASTCALL _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2628 {
2629 #if ZEND_MM_CUSTOM
2630 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2631 _efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2632 return;
2633 }
2634 #endif
2635 zend_mm_free_heap(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2636 }
2637
2638 ZEND_API void* ZEND_FASTCALL _erealloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2639 {
2640 #if ZEND_MM_CUSTOM
2641 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2642 return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2643 }
2644 #endif
2645 return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2646 }
2647
2648 ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2649 {
2650 #if ZEND_MM_CUSTOM
2651 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2652 return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2653 }
2654 #endif
2655 return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2656 }
2657
2658 ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2659 {
2660 return _zend_mm_block_size(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2661 }
2662
2663 ZEND_API void* ZEND_FASTCALL _safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2664 {
2665 return _emalloc(zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2666 }
2667
2668 ZEND_API void* ZEND_FASTCALL _safe_malloc(size_t nmemb, size_t size, size_t offset)
2669 {
2670 return pemalloc(zend_safe_address_guarded(nmemb, size, offset), 1);
2671 }
2672
2673 ZEND_API void* ZEND_FASTCALL _safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2674 {
2675 return _erealloc(ptr, zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2676 }
2677
2678 ZEND_API void* ZEND_FASTCALL _safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset)
2679 {
2680 return perealloc(ptr, zend_safe_address_guarded(nmemb, size, offset), 1);
2681 }
2682
2683 ZEND_API void* ZEND_FASTCALL _ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2684 {
2685 void *p;
2686
2687 size = zend_safe_address_guarded(nmemb, size, 0);
2688 p = _emalloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2689 memset(p, 0, size);
2690 return p;
2691 }
2692
2693 ZEND_API char* ZEND_FASTCALL _estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2694 {
2695 size_t length;
2696 char *p;
2697
2698 length = strlen(s);
2699 if (UNEXPECTED(length + 1 == 0)) {
2700 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2701 }
2702 p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2703 memcpy(p, s, length+1);
2704 return p;
2705 }
2706
2707 ZEND_API char* ZEND_FASTCALL _estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2708 {
2709 char *p;
2710
2711 if (UNEXPECTED(length + 1 == 0)) {
2712 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2713 }
2714 p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2715 memcpy(p, s, length);
2716 p[length] = 0;
2717 return p;
2718 }
2719
2720 static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void);
2721
2722 ZEND_API char* ZEND_FASTCALL zend_strndup(const char *s, size_t length)
2723 {
2724 char *p;
2725
2726 if (UNEXPECTED(length + 1 == 0)) {
2727 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2728 }
2729 p = (char *) malloc(length + 1);
2730 if (UNEXPECTED(p == NULL)) {
2731 zend_out_of_memory();
2732 }
2733 if (EXPECTED(length)) {
2734 memcpy(p, s, length);
2735 }
2736 p[length] = 0;
2737 return p;
2738 }
2739
2740 ZEND_API zend_result zend_set_memory_limit(size_t memory_limit)
2741 {
2742 #if ZEND_MM_LIMIT
2743 zend_mm_heap *heap = AG(mm_heap);
2744
2745 if (UNEXPECTED(memory_limit < heap->real_size)) {
2746 if (memory_limit >= heap->real_size - heap->cached_chunks_count * ZEND_MM_CHUNK_SIZE) {
2747 /* free some cached chunks to fit into new memory limit */
2748 do {
2749 zend_mm_chunk *p = heap->cached_chunks;
2750 heap->cached_chunks = p->next;
2751 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2752 heap->cached_chunks_count--;
2753 heap->real_size -= ZEND_MM_CHUNK_SIZE;
2754 } while (memory_limit < heap->real_size);
2755 return SUCCESS;
2756 }
2757 return FAILURE;
2758 }
2759 AG(mm_heap)->limit = memory_limit;
2760 #endif
2761 return SUCCESS;
2762 }
2763
2764 ZEND_API bool zend_alloc_in_memory_limit_error_reporting(void)
2765 {
2766 #if ZEND_MM_LIMIT
2767 return AG(mm_heap)->overflow;
2768 #else
2769 return false;
2770 #endif
2771 }
2772
2773 ZEND_API size_t zend_memory_usage(bool real_usage)
2774 {
2775 #if ZEND_MM_STAT
2776 if (real_usage) {
2777 return AG(mm_heap)->real_size;
2778 } else {
2779 size_t usage = AG(mm_heap)->size;
2780 return usage;
2781 }
2782 #endif
2783 return 0;
2784 }
2785
2786 ZEND_API size_t zend_memory_peak_usage(bool real_usage)
2787 {
2788 #if ZEND_MM_STAT
2789 if (real_usage) {
2790 return AG(mm_heap)->real_peak;
2791 } else {
2792 return AG(mm_heap)->peak;
2793 }
2794 #endif
2795 return 0;
2796 }
2797
2798 ZEND_API void zend_memory_reset_peak_usage(void)
2799 {
2800 #if ZEND_MM_STAT
2801 AG(mm_heap)->real_peak = AG(mm_heap)->real_size;
2802 AG(mm_heap)->peak = AG(mm_heap)->size;
2803 #endif
2804 }
2805
2806 ZEND_API void shutdown_memory_manager(bool silent, bool full_shutdown)
2807 {
2808 zend_mm_shutdown(AG(mm_heap), full_shutdown, silent);
2809 }
2810
2811 static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void)
2812 {
2813 fprintf(stderr, "Out of memory\n");
2814 exit(1);
2815 }
2816
2817 #if ZEND_MM_CUSTOM
2818 static zend_always_inline void tracked_add(zend_mm_heap *heap, void *ptr, size_t size) {
2819 zval size_zv;
2820 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2821 ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr);
2822 ZVAL_LONG(&size_zv, size);
2823 zend_hash_index_add_new(heap->tracked_allocs, h, &size_zv);
2824 }
2825
2826 static zend_always_inline zval *tracked_get_size_zv(zend_mm_heap *heap, void *ptr) {
2827 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2828 zval *size_zv = zend_hash_index_find(heap->tracked_allocs, h);
2829 ZEND_ASSERT(size_zv && "Trying to free pointer not allocated through ZendMM");
2830 return size_zv;
2831 }
2832
2833 static zend_always_inline void tracked_check_limit(zend_mm_heap *heap, size_t add_size) {
2834 if (add_size > heap->limit - heap->size && !heap->overflow) {
2835 #if ZEND_DEBUG
2836 zend_mm_safe_error(heap,
2837 "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)",
2838 heap->limit, "file", 0, add_size);
2839 #else
2840 zend_mm_safe_error(heap,
2841 "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)",
2842 heap->limit, add_size);
2843 #endif
2844 }
2845 }
2846
2847 static void *tracked_malloc(size_t size)
2848 {
2849 zend_mm_heap *heap = AG(mm_heap);
2850 tracked_check_limit(heap, size);
2851
2852 void *ptr = malloc(size);
2853 if (!ptr) {
2854 zend_out_of_memory();
2855 }
2856
2857 tracked_add(heap, ptr, size);
2858 heap->size += size;
2859 return ptr;
2860 }
2861
2862 static void tracked_free(void *ptr) {
2863 if (!ptr) {
2864 return;
2865 }
2866
2867 zend_mm_heap *heap = AG(mm_heap);
2868 zval *size_zv = tracked_get_size_zv(heap, ptr);
2869 heap->size -= Z_LVAL_P(size_zv);
2870 zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) size_zv);
2871 free(ptr);
2872 }
2873
2874 static void *tracked_realloc(void *ptr, size_t new_size) {
2875 zend_mm_heap *heap = AG(mm_heap);
2876 zval *old_size_zv = NULL;
2877 size_t old_size = 0;
2878 if (ptr) {
2879 old_size_zv = tracked_get_size_zv(heap, ptr);
2880 old_size = Z_LVAL_P(old_size_zv);
2881 }
2882
2883 if (new_size > old_size) {
2884 tracked_check_limit(heap, new_size - old_size);
2885 }
2886
2887 /* Delete information about old allocation only after checking the memory limit. */
2888 if (old_size_zv) {
2889 zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) old_size_zv);
2890 }
2891
2892 ptr = __zend_realloc(ptr, new_size);
2893 tracked_add(heap, ptr, new_size);
2894 heap->size += new_size - old_size;
2895 return ptr;
2896 }
2897
2898 static void tracked_free_all(void) {
2899 HashTable *tracked_allocs = AG(mm_heap)->tracked_allocs;
2900 zend_ulong h;
2901 ZEND_HASH_FOREACH_NUM_KEY(tracked_allocs, h) {
2902 void *ptr = (void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2);
2903 free(ptr);
2904 } ZEND_HASH_FOREACH_END();
2905 }
2906 #endif
2907
2908 static void alloc_globals_ctor(zend_alloc_globals *alloc_globals)
2909 {
2910 char *tmp;
2911
2912 #if ZEND_MM_CUSTOM
2913 tmp = getenv("USE_ZEND_ALLOC");
2914 if (tmp && !ZEND_ATOL(tmp)) {
2915 bool tracked = (tmp = getenv("USE_TRACKED_ALLOC")) && ZEND_ATOL(tmp);
2916 zend_mm_heap *mm_heap = alloc_globals->mm_heap = malloc(sizeof(zend_mm_heap));
2917 memset(mm_heap, 0, sizeof(zend_mm_heap));
2918 mm_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2919 mm_heap->limit = (size_t)Z_L(-1) >> 1;
2920 mm_heap->overflow = 0;
2921
2922 if (!tracked) {
2923 /* Use system allocator. */
2924 mm_heap->custom_heap.std._malloc = __zend_malloc;
2925 mm_heap->custom_heap.std._free = free;
2926 mm_heap->custom_heap.std._realloc = __zend_realloc;
2927 } else {
2928 /* Use system allocator and track allocations for auto-free. */
2929 mm_heap->custom_heap.std._malloc = tracked_malloc;
2930 mm_heap->custom_heap.std._free = tracked_free;
2931 mm_heap->custom_heap.std._realloc = tracked_realloc;
2932 mm_heap->tracked_allocs = malloc(sizeof(HashTable));
2933 zend_hash_init(mm_heap->tracked_allocs, 1024, NULL, NULL, 1);
2934 }
2935 return;
2936 }
2937 #endif
2938
2939 tmp = getenv("USE_ZEND_ALLOC_HUGE_PAGES");
2940 if (tmp && ZEND_ATOL(tmp)) {
2941 zend_mm_use_huge_pages = true;
2942 }
2943 alloc_globals->mm_heap = zend_mm_init();
2944 }
2945
2946 #ifdef ZTS
2947 static void alloc_globals_dtor(zend_alloc_globals *alloc_globals)
2948 {
2949 zend_mm_shutdown(alloc_globals->mm_heap, 1, 1);
2950 }
2951 #endif
2952
2953 ZEND_API void start_memory_manager(void)
2954 {
2955 #ifdef ZTS
2956 ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
2957 #else
2958 alloc_globals_ctor(&alloc_globals);
2959 #endif
2960 #ifndef _WIN32
2961 # if defined(_SC_PAGESIZE)
2962 REAL_PAGE_SIZE = sysconf(_SC_PAGESIZE);
2963 # elif defined(_SC_PAGE_SIZE)
2964 REAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
2965 # endif
2966 #endif
2967 }
2968
2969 ZEND_API zend_mm_heap *zend_mm_set_heap(zend_mm_heap *new_heap)
2970 {
2971 zend_mm_heap *old_heap;
2972
2973 old_heap = AG(mm_heap);
2974 AG(mm_heap) = (zend_mm_heap*)new_heap;
2975 return (zend_mm_heap*)old_heap;
2976 }
2977
2978 ZEND_API zend_mm_heap *zend_mm_get_heap(void)
2979 {
2980 return AG(mm_heap);
2981 }
2982
2983 ZEND_API bool zend_mm_is_custom_heap(zend_mm_heap *new_heap)
2984 {
2985 #if ZEND_MM_CUSTOM
2986 return AG(mm_heap)->use_custom_heap;
2987 #else
2988 return 0;
2989 #endif
2990 }
2991
2992 ZEND_API void zend_mm_set_custom_handlers(zend_mm_heap *heap,
2993 void* (*_malloc)(size_t),
2994 void (*_free)(void*),
2995 void* (*_realloc)(void*, size_t))
2996 {
2997 #if ZEND_MM_CUSTOM
2998 zend_mm_heap *_heap = (zend_mm_heap*)heap;
2999
3000 if (!_malloc && !_free && !_realloc) {
3001 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
3002 } else {
3003 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
3004 _heap->custom_heap.std._malloc = _malloc;
3005 _heap->custom_heap.std._free = _free;
3006 _heap->custom_heap.std._realloc = _realloc;
3007 }
3008 #endif
3009 }
3010
3011 ZEND_API void zend_mm_get_custom_handlers(zend_mm_heap *heap,
3012 void* (**_malloc)(size_t),
3013 void (**_free)(void*),
3014 void* (**_realloc)(void*, size_t))
3015 {
3016 #if ZEND_MM_CUSTOM
3017 zend_mm_heap *_heap = (zend_mm_heap*)heap;
3018
3019 if (heap->use_custom_heap) {
3020 *_malloc = _heap->custom_heap.std._malloc;
3021 *_free = _heap->custom_heap.std._free;
3022 *_realloc = _heap->custom_heap.std._realloc;
3023 } else {
3024 *_malloc = NULL;
3025 *_free = NULL;
3026 *_realloc = NULL;
3027 }
3028 #else
3029 *_malloc = NULL;
3030 *_free = NULL;
3031 *_realloc = NULL;
3032 #endif
3033 }
3034
3035 #if ZEND_DEBUG
3036 ZEND_API void zend_mm_set_custom_debug_handlers(zend_mm_heap *heap,
3037 void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
3038 void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
3039 void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC))
3040 {
3041 #if ZEND_MM_CUSTOM
3042 zend_mm_heap *_heap = (zend_mm_heap*)heap;
3043
3044 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_DEBUG;
3045 _heap->custom_heap.debug._malloc = _malloc;
3046 _heap->custom_heap.debug._free = _free;
3047 _heap->custom_heap.debug._realloc = _realloc;
3048 #endif
3049 }
3050 #endif
3051
3052 ZEND_API zend_mm_storage *zend_mm_get_storage(zend_mm_heap *heap)
3053 {
3054 #if ZEND_MM_STORAGE
3055 return heap->storage;
3056 #else
3057 return NULL
3058 #endif
3059 }
3060
3061 ZEND_API zend_mm_heap *zend_mm_startup(void)
3062 {
3063 return zend_mm_init();
3064 }
3065
3066 ZEND_API zend_mm_heap *zend_mm_startup_ex(const zend_mm_handlers *handlers, void *data, size_t data_size)
3067 {
3068 #if ZEND_MM_STORAGE
3069 zend_mm_storage tmp_storage, *storage;
3070 zend_mm_chunk *chunk;
3071 zend_mm_heap *heap;
3072
3073 memcpy((zend_mm_handlers*)&tmp_storage.handlers, handlers, sizeof(zend_mm_handlers));
3074 tmp_storage.data = data;
3075 chunk = (zend_mm_chunk*)handlers->chunk_alloc(&tmp_storage, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
3076 if (UNEXPECTED(chunk == NULL)) {
3077 #if ZEND_MM_ERROR
3078 fprintf(stderr, "Can't initialize heap\n");
3079 #endif
3080 return NULL;
3081 }
3082 heap = &chunk->heap_slot;
3083 chunk->heap = heap;
3084 chunk->next = chunk;
3085 chunk->prev = chunk;
3086 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
3087 chunk->free_tail = ZEND_MM_FIRST_PAGE;
3088 chunk->num = 0;
3089 chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
3090 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
3091 heap->main_chunk = chunk;
3092 heap->cached_chunks = NULL;
3093 heap->chunks_count = 1;
3094 heap->peak_chunks_count = 1;
3095 heap->cached_chunks_count = 0;
3096 heap->avg_chunks_count = 1.0;
3097 heap->last_chunks_delete_boundary = 0;
3098 heap->last_chunks_delete_count = 0;
3099 #if ZEND_MM_STAT || ZEND_MM_LIMIT
3100 heap->real_size = ZEND_MM_CHUNK_SIZE;
3101 #endif
3102 #if ZEND_MM_STAT
3103 heap->real_peak = ZEND_MM_CHUNK_SIZE;
3104 heap->size = 0;
3105 heap->peak = 0;
3106 #endif
3107 #if ZEND_MM_LIMIT
3108 heap->limit = (size_t)Z_L(-1) >> 1;
3109 heap->overflow = 0;
3110 #endif
3111 #if ZEND_MM_CUSTOM
3112 heap->use_custom_heap = 0;
3113 #endif
3114 heap->storage = &tmp_storage;
3115 heap->huge_list = NULL;
3116 memset(heap->free_slot, 0, sizeof(heap->free_slot));
3117 storage = _zend_mm_alloc(heap, sizeof(zend_mm_storage) + data_size ZEND_FILE_LINE_CC ZEND_FILE_LINE_CC);
3118 if (!storage) {
3119 handlers->chunk_free(&tmp_storage, chunk, ZEND_MM_CHUNK_SIZE);
3120 #if ZEND_MM_ERROR
3121 fprintf(stderr, "Can't initialize heap\n");
3122 #endif
3123 return NULL;
3124 }
3125 memcpy(storage, &tmp_storage, sizeof(zend_mm_storage));
3126 if (data) {
3127 storage->data = (void*)(((char*)storage + sizeof(zend_mm_storage)));
3128 memcpy(storage->data, data, data_size);
3129 }
3130 heap->storage = storage;
3131 return heap;
3132 #else
3133 return NULL;
3134 #endif
3135 }
3136
3137 ZEND_API void * __zend_malloc(size_t len)
3138 {
3139 void *tmp = malloc(len);
3140 if (EXPECTED(tmp || !len)) {
3141 return tmp;
3142 }
3143 zend_out_of_memory();
3144 }
3145
3146 ZEND_API void * __zend_calloc(size_t nmemb, size_t len)
3147 {
3148 void *tmp;
3149
3150 len = zend_safe_address_guarded(nmemb, len, 0);
3151 tmp = __zend_malloc(len);
3152 memset(tmp, 0, len);
3153 return tmp;
3154 }
3155
3156 ZEND_API void * __zend_realloc(void *p, size_t len)
3157 {
3158 p = realloc(p, len);
3159 if (EXPECTED(p || !len)) {
3160 return p;
3161 }
3162 zend_out_of_memory();
3163 }
3164
3165 ZEND_API char * __zend_strdup(const char *s)
3166 {
3167 char *tmp = strdup(s);
3168 if (EXPECTED(tmp)) {
3169 return tmp;
3170 }
3171 zend_out_of_memory();
3172 }
3173
3174 #ifdef ZTS
3175 size_t zend_mm_globals_size(void)
3176 {
3177 return sizeof(zend_alloc_globals);
3178 }
3179 #endif
3180
