1 /*
2 +----------------------------------------------------------------------+
3 | Zend Engine |
4 +----------------------------------------------------------------------+
5 | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 2.00 of the Zend license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.zend.com/license/2_00.txt. |
11 | If you did not receive a copy of the Zend license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@zend.com so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Andi Gutmans <andi@php.net> |
16 | Zeev Suraski <zeev@php.net> |
17 | Dmitry Stogov <dmitry@php.net> |
18 +----------------------------------------------------------------------+
19 */
20
21 /*
22 * zend_alloc is designed to be a modern CPU cache friendly memory manager
23 * for PHP. Most ideas are taken from jemalloc and tcmalloc implementations.
24 *
25 * All allocations are split into 3 categories:
26 *
27 * Huge - the size is greater than CHUNK size (~2M by default), allocation is
28 * performed using mmap(). The result is aligned on 2M boundary.
29 *
30 * Large - a number of 4096K pages inside a CHUNK. Large blocks
31 * are always aligned on page boundary.
32 *
33 * Small - less than 3/4 of page size. Small sizes are rounded up to nearest
34 * greater predefined small size (there are 30 predefined sizes:
35 * 8, 16, 24, 32, ... 3072). Small blocks are allocated from
36 * RUNs. Each RUN is allocated as a single or few following pages.
37 * Allocation inside RUNs implemented using linked list of free
38 * elements. The result is aligned to 8 bytes.
39 *
40 * zend_alloc allocates memory from OS by CHUNKs, these CHUNKs and huge memory
41 * blocks are always aligned to CHUNK boundary. So it's very easy to determine
42 * the CHUNK owning the certain pointer. Regular CHUNKs reserve a single
43 * page at start for special purpose. It contains bitset of free pages,
44 * few bitset for available runs of predefined small sizes, map of pages that
45 * keeps information about usage of each page in this CHUNK, etc.
46 *
47 * zend_alloc provides familiar emalloc/efree/erealloc API, but in addition it
48 * provides specialized and optimized routines to allocate blocks of predefined
49 * sizes (e.g. emalloc_2(), emallc_4(), ..., emalloc_large(), etc)
50 * The library uses C preprocessor tricks that substitute calls to emalloc()
51 * with more specialized routines when the requested size is known.
52 */
53
54 #include "zend.h"
55 #include "zend_alloc.h"
56 #include "zend_globals.h"
57 #include "zend_operators.h"
58 #include "zend_multiply.h"
59 #include "zend_bitset.h"
60 #include "zend_mmap.h"
61 #include <signal.h>
62
63 #ifdef HAVE_UNISTD_H
64 # include <unistd.h>
65 #endif
66
67 #ifdef ZEND_WIN32
68 # include <wincrypt.h>
69 # include <process.h>
70 # include "win32/winutil.h"
71 #endif
72
73 #include <stdio.h>
74 #include <stdlib.h>
75 #include <string.h>
76
77 #include <sys/types.h>
78 #include <sys/stat.h>
79 #include <limits.h>
80 #include <fcntl.h>
81 #include <errno.h>
82 #ifdef __SANITIZE_ADDRESS__
83 # include <sanitizer/asan_interface.h>
84 #endif
85
86 #ifndef _WIN32
87 # include <sys/mman.h>
88 # ifndef MAP_ANON
89 # ifdef MAP_ANONYMOUS
90 # define MAP_ANON MAP_ANONYMOUS
91 # endif
92 # endif
93 # ifndef MAP_FAILED
94 # define MAP_FAILED ((void*)-1)
95 # endif
96 # ifndef MAP_POPULATE
97 # define MAP_POPULATE 0
98 # endif
99 # if defined(_SC_PAGESIZE) || (_SC_PAGE_SIZE)
100 # define REAL_PAGE_SIZE _real_page_size
101 static size_t _real_page_size = ZEND_MM_PAGE_SIZE;
102 # endif
103 # ifdef MAP_ALIGNED_SUPER
104 # define MAP_HUGETLB MAP_ALIGNED_SUPER
105 # endif
106 #endif
107
108 #ifndef REAL_PAGE_SIZE
109 # define REAL_PAGE_SIZE ZEND_MM_PAGE_SIZE
110 #endif
111
112 /* NetBSD has an mremap() function with a signature that is incompatible with Linux (WTF?),
113 * so pretend it doesn't exist. */
114 #ifndef __linux__
115 # undef HAVE_MREMAP
116 #endif
117
118 #ifndef __APPLE__
119 # define ZEND_MM_FD -1
120 #else
121 # include <mach/vm_statistics.h>
122 /* Mac allows to track anonymous page via vmmap per TAG id.
123 * user land applications are allowed to take from 240 to 255.
124 */
125 # define ZEND_MM_FD VM_MAKE_TAG(250U)
126 #endif
127
128 #ifndef ZEND_MM_STAT
129 # define ZEND_MM_STAT 1 /* track current and peak memory usage */
130 #endif
131 #ifndef ZEND_MM_LIMIT
132 # define ZEND_MM_LIMIT 1 /* support for user-defined memory limit */
133 #endif
134 #ifndef ZEND_MM_CUSTOM
135 # define ZEND_MM_CUSTOM 1 /* support for custom memory allocator */
136 /* USE_ZEND_ALLOC=0 may switch to system malloc() */
137 #endif
138 #ifndef ZEND_MM_STORAGE
139 # define ZEND_MM_STORAGE 1 /* support for custom memory storage */
140 #endif
141 #ifndef ZEND_MM_ERROR
142 # define ZEND_MM_ERROR 1 /* report system errors */
143 #endif
144
145 #ifndef ZEND_MM_CHECK
146 # define ZEND_MM_CHECK(condition, message) do { \
147 if (UNEXPECTED(!(condition))) { \
148 zend_mm_panic(message); \
149 } \
150 } while (0)
151 #endif
152
153 typedef uint32_t zend_mm_page_info; /* 4-byte integer */
154 typedef zend_ulong zend_mm_bitset; /* 4-byte or 8-byte integer */
155
156 #define ZEND_MM_ALIGNED_OFFSET(size, alignment) \
157 (((size_t)(size)) & ((alignment) - 1))
158 #define ZEND_MM_ALIGNED_BASE(size, alignment) \
159 (((size_t)(size)) & ~((alignment) - 1))
160 #define ZEND_MM_SIZE_TO_NUM(size, alignment) \
161 (((size_t)(size) + ((alignment) - 1)) / (alignment))
162
163 #define ZEND_MM_BITSET_LEN (sizeof(zend_mm_bitset) * 8) /* 32 or 64 */
164 #define ZEND_MM_PAGE_MAP_LEN (ZEND_MM_PAGES / ZEND_MM_BITSET_LEN) /* 16 or 8 */
165
166 typedef zend_mm_bitset zend_mm_page_map[ZEND_MM_PAGE_MAP_LEN]; /* 64B */
167
168 #define ZEND_MM_IS_FRUN 0x00000000
169 #define ZEND_MM_IS_LRUN 0x40000000
170 #define ZEND_MM_IS_SRUN 0x80000000
171
172 #define ZEND_MM_LRUN_PAGES_MASK 0x000003ff
173 #define ZEND_MM_LRUN_PAGES_OFFSET 0
174
175 #define ZEND_MM_SRUN_BIN_NUM_MASK 0x0000001f
176 #define ZEND_MM_SRUN_BIN_NUM_OFFSET 0
177
178 #define ZEND_MM_SRUN_FREE_COUNTER_MASK 0x01ff0000
179 #define ZEND_MM_SRUN_FREE_COUNTER_OFFSET 16
180
181 #define ZEND_MM_NRUN_OFFSET_MASK 0x01ff0000
182 #define ZEND_MM_NRUN_OFFSET_OFFSET 16
183
184 #define ZEND_MM_LRUN_PAGES(info) (((info) & ZEND_MM_LRUN_PAGES_MASK) >> ZEND_MM_LRUN_PAGES_OFFSET)
185 #define ZEND_MM_SRUN_BIN_NUM(info) (((info) & ZEND_MM_SRUN_BIN_NUM_MASK) >> ZEND_MM_SRUN_BIN_NUM_OFFSET)
186 #define ZEND_MM_SRUN_FREE_COUNTER(info) (((info) & ZEND_MM_SRUN_FREE_COUNTER_MASK) >> ZEND_MM_SRUN_FREE_COUNTER_OFFSET)
187 #define ZEND_MM_NRUN_OFFSET(info) (((info) & ZEND_MM_NRUN_OFFSET_MASK) >> ZEND_MM_NRUN_OFFSET_OFFSET)
188
189 #define ZEND_MM_FRUN() ZEND_MM_IS_FRUN
190 #define ZEND_MM_LRUN(count) (ZEND_MM_IS_LRUN | ((count) << ZEND_MM_LRUN_PAGES_OFFSET))
191 #define ZEND_MM_SRUN(bin_num) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET))
192 #define ZEND_MM_SRUN_EX(bin_num, count) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((count) << ZEND_MM_SRUN_FREE_COUNTER_OFFSET))
193 #define ZEND_MM_NRUN(bin_num, offset) (ZEND_MM_IS_SRUN | ZEND_MM_IS_LRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((offset) << ZEND_MM_NRUN_OFFSET_OFFSET))
194
195 #define ZEND_MM_BINS 30
196
197 typedef struct _zend_mm_page zend_mm_page;
198 typedef struct _zend_mm_bin zend_mm_bin;
199 typedef struct _zend_mm_free_slot zend_mm_free_slot;
200 typedef struct _zend_mm_chunk zend_mm_chunk;
201 typedef struct _zend_mm_huge_list zend_mm_huge_list;
202
203 static bool zend_mm_use_huge_pages = false;
204
205 /*
206 * Memory is retrieved from OS by chunks of fixed size 2MB.
207 * Inside chunk it's managed by pages of fixed size 4096B.
208 * So each chunk consists from 512 pages.
209 * The first page of each chunk is reserved for chunk header.
210 * It contains service information about all pages.
211 *
212 * free_pages - current number of free pages in this chunk
213 *
214 * free_tail - number of continuous free pages at the end of chunk
215 *
216 * free_map - bitset (a bit for each page). The bit is set if the corresponding
217 * page is allocated. Allocator for "large sizes" may easily find a
218 * free page (or a continuous number of pages) searching for zero
219 * bits.
220 *
221 * map - contains service information for each page. (32-bits for each
222 * page).
223 * usage:
224 * (2 bits)
225 * FRUN - free page,
226 * LRUN - first page of "large" allocation
227 * SRUN - first page of a bin used for "small" allocation
228 *
229 * lrun_pages:
230 * (10 bits) number of allocated pages
231 *
232 * srun_bin_num:
233 * (5 bits) bin number (e.g. 0 for sizes 0-2, 1 for 3-4,
234 * 2 for 5-8, 3 for 9-16 etc) see zend_alloc_sizes.h
235 */
236
237 struct _zend_mm_heap {
238 #if ZEND_MM_CUSTOM
239 int use_custom_heap;
240 #endif
241 #if ZEND_MM_STORAGE
242 zend_mm_storage *storage;
243 #endif
244 #if ZEND_MM_STAT
245 size_t size; /* current memory usage */
246 size_t peak; /* peak memory usage */
247 #endif
248 zend_mm_free_slot *free_slot[ZEND_MM_BINS]; /* free lists for small sizes */
249 #if ZEND_MM_STAT || ZEND_MM_LIMIT
250 size_t real_size; /* current size of allocated pages */
251 #endif
252 #if ZEND_MM_STAT
253 size_t real_peak; /* peak size of allocated pages */
254 #endif
255 #if ZEND_MM_LIMIT
256 size_t limit; /* memory limit */
257 int overflow; /* memory overflow flag */
258 #endif
259
260 zend_mm_huge_list *huge_list; /* list of huge allocated blocks */
261
262 zend_mm_chunk *main_chunk;
263 zend_mm_chunk *cached_chunks; /* list of unused chunks */
264 int chunks_count; /* number of allocated chunks */
265 int peak_chunks_count; /* peak number of allocated chunks for current request */
266 int cached_chunks_count; /* number of cached chunks */
267 double avg_chunks_count; /* average number of chunks allocated per request */
268 int last_chunks_delete_boundary; /* number of chunks after last deletion */
269 int last_chunks_delete_count; /* number of deletion over the last boundary */
270 #if ZEND_MM_CUSTOM
271 struct {
272 void *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
273 void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
274 void *(*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
275 } custom_heap;
276 HashTable *tracked_allocs;
277 #endif
278 };
279
280 struct _zend_mm_chunk {
281 zend_mm_heap *heap;
282 zend_mm_chunk *next;
283 zend_mm_chunk *prev;
284 uint32_t free_pages; /* number of free pages */
285 uint32_t free_tail; /* number of free pages at the end of chunk */
286 uint32_t num;
287 char reserve[64 - (sizeof(void*) * 3 + sizeof(uint32_t) * 3)];
288 zend_mm_heap heap_slot; /* used only in main chunk */
289 zend_mm_page_map free_map; /* 512 bits or 64 bytes */
290 zend_mm_page_info map[ZEND_MM_PAGES]; /* 2 KB = 512 * 4 */
291 };
292
293 struct _zend_mm_page {
294 char bytes[ZEND_MM_PAGE_SIZE];
295 };
296
297 /*
298 * bin - is one or few continuous pages (up to 8) used for allocation of
299 * a particular "small size".
300 */
301 struct _zend_mm_bin {
302 char bytes[ZEND_MM_PAGE_SIZE * 8];
303 };
304
305 struct _zend_mm_free_slot {
306 zend_mm_free_slot *next_free_slot;
307 };
308
309 struct _zend_mm_huge_list {
310 void *ptr;
311 size_t size;
312 zend_mm_huge_list *next;
313 #if ZEND_DEBUG
314 zend_mm_debug_info dbg;
315 #endif
316 };
317
318 #define ZEND_MM_PAGE_ADDR(chunk, page_num) \
319 ((void*)(((zend_mm_page*)(chunk)) + (page_num)))
320
321 #define _BIN_DATA_SIZE(num, size, elements, pages, x, y) size,
322 static const uint32_t bin_data_size[] = {
323 ZEND_MM_BINS_INFO(_BIN_DATA_SIZE, x, y)
324 };
325
326 #define _BIN_DATA_ELEMENTS(num, size, elements, pages, x, y) elements,
327 static const uint32_t bin_elements[] = {
328 ZEND_MM_BINS_INFO(_BIN_DATA_ELEMENTS, x, y)
329 };
330
331 #define _BIN_DATA_PAGES(num, size, elements, pages, x, y) pages,
332 static const uint32_t bin_pages[] = {
333 ZEND_MM_BINS_INFO(_BIN_DATA_PAGES, x, y)
334 };
335
336 #if ZEND_DEBUG
zend_debug_alloc_output(char * format,...)337 ZEND_COLD void zend_debug_alloc_output(char *format, ...)
338 {
339 char output_buf[256];
340 va_list args;
341
342 va_start(args, format);
343 vsprintf(output_buf, format, args);
344 va_end(args);
345
346 #ifdef ZEND_WIN32
347 OutputDebugString(output_buf);
348 #else
349 fprintf(stderr, "%s", output_buf);
350 #endif
351 }
352 #endif
353
zend_mm_panic(const char * message)354 static ZEND_COLD ZEND_NORETURN void zend_mm_panic(const char *message)
355 {
356 fprintf(stderr, "%s\n", message);
357 /* See http://support.microsoft.com/kb/190351 */
358 #ifdef ZEND_WIN32
359 fflush(stderr);
360 #endif
361 #if ZEND_DEBUG && defined(HAVE_KILL) && defined(HAVE_GETPID)
362 kill(getpid(), SIGSEGV);
363 #endif
364 abort();
365 }
366
zend_mm_safe_error(zend_mm_heap * heap,const char * format,size_t limit,const char * filename,uint32_t lineno,size_t size)367 static ZEND_COLD ZEND_NORETURN void zend_mm_safe_error(zend_mm_heap *heap,
368 const char *format,
369 size_t limit,
370 #if ZEND_DEBUG
371 const char *filename,
372 uint32_t lineno,
373 #endif
374 size_t size)
375 {
376
377 heap->overflow = 1;
378 zend_try {
379 zend_error_noreturn(E_ERROR,
380 format,
381 limit,
382 #if ZEND_DEBUG
383 filename,
384 lineno,
385 #endif
386 size);
387 } zend_catch {
388 } zend_end_try();
389 heap->overflow = 0;
390 zend_bailout();
391 exit(1);
392 }
393
394 #ifdef _WIN32
stderr_last_error(char * msg)395 static void stderr_last_error(char *msg)
396 {
397 DWORD err = GetLastError();
398 char *buf = php_win32_error_to_msg(err);
399
400 if (!buf[0]) {
401 fprintf(stderr, "\n%s: [0x%08lx]\n", msg, err);
402 }
403 else {
404 fprintf(stderr, "\n%s: [0x%08lx] %s\n", msg, err, buf);
405 }
406
407 php_win32_error_msg_free(buf);
408 }
409 #endif
410
411 /*****************/
412 /* OS Allocation */
413 /*****************/
414
zend_mm_munmap(void * addr,size_t size)415 static void zend_mm_munmap(void *addr, size_t size)
416 {
417 #ifdef _WIN32
418 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
419 /** ERROR_INVALID_ADDRESS is expected when addr is not range start address */
420 if (GetLastError() != ERROR_INVALID_ADDRESS) {
421 #if ZEND_MM_ERROR
422 stderr_last_error("VirtualFree() failed");
423 #endif
424 return;
425 }
426 SetLastError(0);
427
428 MEMORY_BASIC_INFORMATION mbi;
429 if (VirtualQuery(addr, &mbi, sizeof(mbi)) == 0) {
430 #if ZEND_MM_ERROR
431 stderr_last_error("VirtualQuery() failed");
432 #endif
433 return;
434 }
435 addr = mbi.AllocationBase;
436
437 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
438 #if ZEND_MM_ERROR
439 stderr_last_error("VirtualFree() failed");
440 #endif
441 }
442 }
443 #else
444 if (munmap(addr, size) != 0) {
445 #if ZEND_MM_ERROR
446 fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno));
447 #endif
448 }
449 #endif
450 }
451
452 #ifndef HAVE_MREMAP
zend_mm_mmap_fixed(void * addr,size_t size)453 static void *zend_mm_mmap_fixed(void *addr, size_t size)
454 {
455 #ifdef _WIN32
456 void *ptr = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
457
458 if (ptr == NULL) {
459 /** ERROR_INVALID_ADDRESS is expected when fixed addr range is not free */
460 if (GetLastError() != ERROR_INVALID_ADDRESS) {
461 #if ZEND_MM_ERROR
462 stderr_last_error("VirtualAlloc() fixed failed");
463 #endif
464 }
465 SetLastError(0);
466 return NULL;
467 }
468 ZEND_ASSERT(ptr == addr);
469 return ptr;
470 #else
471 int flags = MAP_PRIVATE | MAP_ANON;
472 #if defined(MAP_EXCL)
473 flags |= MAP_FIXED | MAP_EXCL;
474 #elif defined(MAP_TRYFIXED)
475 flags |= MAP_TRYFIXED;
476 #endif
477 /* MAP_FIXED leads to discarding of the old mapping, so it can't be used. */
478 void *ptr = mmap(addr, size, PROT_READ | PROT_WRITE, flags /*| MAP_POPULATE | MAP_HUGETLB*/, ZEND_MM_FD, 0);
479
480 if (ptr == MAP_FAILED) {
481 #if ZEND_MM_ERROR && !defined(MAP_EXCL) && !defined(MAP_TRYFIXED)
482 fprintf(stderr, "\nmmap() fixed failed: [%d] %s\n", errno, strerror(errno));
483 #endif
484 return NULL;
485 } else if (ptr != addr) {
486 zend_mm_munmap(ptr, size);
487 return NULL;
488 }
489 return ptr;
490 #endif
491 }
492 #endif
493
zend_mm_mmap(size_t size)494 static void *zend_mm_mmap(size_t size)
495 {
496 #ifdef _WIN32
497 void *ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
498
499 if (ptr == NULL) {
500 #if ZEND_MM_ERROR
501 stderr_last_error("VirtualAlloc() failed");
502 #endif
503 return NULL;
504 }
505 return ptr;
506 #else
507 void *ptr;
508
509 #if defined(MAP_HUGETLB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB)
510 if (zend_mm_use_huge_pages && size == ZEND_MM_CHUNK_SIZE) {
511 int fd = -1;
512 int mflags = MAP_PRIVATE | MAP_ANON;
513 #if defined(MAP_HUGETLB)
514 mflags |= MAP_HUGETLB;
515 #else
516 fd = VM_FLAGS_SUPERPAGE_SIZE_2MB;
517 #endif
518 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, mflags, fd, 0);
519 if (ptr != MAP_FAILED) {
520 zend_mmap_set_name(ptr, size, "zend_alloc");
521 return ptr;
522 }
523 }
524 #endif
525
526 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, ZEND_MM_FD, 0);
527
528 if (ptr == MAP_FAILED) {
529 #if ZEND_MM_ERROR
530 fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno));
531 #endif
532 return NULL;
533 }
534 zend_mmap_set_name(ptr, size, "zend_alloc");
535 return ptr;
536 #endif
537 }
538
539 /***********/
540 /* Bitmask */
541 /***********/
542
543 /* number of trailing set (1) bits */
zend_mm_bitset_nts(zend_mm_bitset bitset)544 ZEND_ATTRIBUTE_CONST static zend_always_inline int zend_mm_bitset_nts(zend_mm_bitset bitset)
545 {
546 #if (defined(__GNUC__) || __has_builtin(__builtin_ctzl)) && SIZEOF_ZEND_LONG == SIZEOF_LONG && defined(PHP_HAVE_BUILTIN_CTZL)
547 return __builtin_ctzl(~bitset);
548 #elif (defined(__GNUC__) || __has_builtin(__builtin_ctzll)) && defined(PHP_HAVE_BUILTIN_CTZLL)
549 return __builtin_ctzll(~bitset);
550 #elif defined(_WIN32)
551 unsigned long index;
552
553 #if defined(_WIN64)
554 if (!BitScanForward64(&index, ~bitset)) {
555 #else
556 if (!BitScanForward(&index, ~bitset)) {
557 #endif
558 /* undefined behavior */
559 return 32;
560 }
561
562 return (int)index;
563 #else
564 int n;
565
566 if (bitset == (zend_mm_bitset)-1) return ZEND_MM_BITSET_LEN;
567
568 n = 0;
569 #if SIZEOF_ZEND_LONG == 8
570 if (sizeof(zend_mm_bitset) == 8) {
571 if ((bitset & 0xffffffff) == 0xffffffff) {n += 32; bitset = bitset >> Z_UL(32);}
572 }
573 #endif
574 if ((bitset & 0x0000ffff) == 0x0000ffff) {n += 16; bitset = bitset >> 16;}
575 if ((bitset & 0x000000ff) == 0x000000ff) {n += 8; bitset = bitset >> 8;}
576 if ((bitset & 0x0000000f) == 0x0000000f) {n += 4; bitset = bitset >> 4;}
577 if ((bitset & 0x00000003) == 0x00000003) {n += 2; bitset = bitset >> 2;}
578 return n + (bitset & 1);
579 #endif
580 }
581
582 static zend_always_inline int zend_mm_bitset_is_set(zend_mm_bitset *bitset, int bit)
583 {
584 return ZEND_BIT_TEST(bitset, bit);
585 }
586
587 static zend_always_inline void zend_mm_bitset_set_bit(zend_mm_bitset *bitset, int bit)
588 {
589 bitset[bit / ZEND_MM_BITSET_LEN] |= (Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
590 }
591
592 static zend_always_inline void zend_mm_bitset_reset_bit(zend_mm_bitset *bitset, int bit)
593 {
594 bitset[bit / ZEND_MM_BITSET_LEN] &= ~(Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
595 }
596
597 static zend_always_inline void zend_mm_bitset_set_range(zend_mm_bitset *bitset, int start, int len)
598 {
599 if (len == 1) {
600 zend_mm_bitset_set_bit(bitset, start);
601 } else {
602 int pos = start / ZEND_MM_BITSET_LEN;
603 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
604 int bit = start & (ZEND_MM_BITSET_LEN - 1);
605 zend_mm_bitset tmp;
606
607 if (pos != end) {
608 /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
609 tmp = (zend_mm_bitset)-1 << bit;
610 bitset[pos++] |= tmp;
611 while (pos != end) {
612 /* set all bits */
613 bitset[pos++] = (zend_mm_bitset)-1;
614 }
615 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
616 /* set bits from "0" to "end" */
617 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
618 bitset[pos] |= tmp;
619 } else {
620 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
621 /* set bits from "bit" to "end" */
622 tmp = (zend_mm_bitset)-1 << bit;
623 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
624 bitset[pos] |= tmp;
625 }
626 }
627 }
628
629 static zend_always_inline void zend_mm_bitset_reset_range(zend_mm_bitset *bitset, int start, int len)
630 {
631 if (len == 1) {
632 zend_mm_bitset_reset_bit(bitset, start);
633 } else {
634 int pos = start / ZEND_MM_BITSET_LEN;
635 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
636 int bit = start & (ZEND_MM_BITSET_LEN - 1);
637 zend_mm_bitset tmp;
638
639 if (pos != end) {
640 /* reset bits from "bit" to ZEND_MM_BITSET_LEN-1 */
641 tmp = ~((Z_UL(1) << bit) - 1);
642 bitset[pos++] &= ~tmp;
643 while (pos != end) {
644 /* set all bits */
645 bitset[pos++] = 0;
646 }
647 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
648 /* reset bits from "0" to "end" */
649 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
650 bitset[pos] &= ~tmp;
651 } else {
652 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
653 /* reset bits from "bit" to "end" */
654 tmp = (zend_mm_bitset)-1 << bit;
655 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
656 bitset[pos] &= ~tmp;
657 }
658 }
659 }
660
661 static zend_always_inline int zend_mm_bitset_is_free_range(zend_mm_bitset *bitset, int start, int len)
662 {
663 if (len == 1) {
664 return !zend_mm_bitset_is_set(bitset, start);
665 } else {
666 int pos = start / ZEND_MM_BITSET_LEN;
667 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
668 int bit = start & (ZEND_MM_BITSET_LEN - 1);
669 zend_mm_bitset tmp;
670
671 if (pos != end) {
672 /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
673 tmp = (zend_mm_bitset)-1 << bit;
674 if ((bitset[pos++] & tmp) != 0) {
675 return 0;
676 }
677 while (pos != end) {
678 /* set all bits */
679 if (bitset[pos++] != 0) {
680 return 0;
681 }
682 }
683 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
684 /* set bits from "0" to "end" */
685 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
686 return (bitset[pos] & tmp) == 0;
687 } else {
688 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
689 /* set bits from "bit" to "end" */
690 tmp = (zend_mm_bitset)-1 << bit;
691 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
692 return (bitset[pos] & tmp) == 0;
693 }
694 }
695 }
696
697 /**********/
698 /* Chunks */
699 /**********/
700
701 static zend_always_inline void zend_mm_hugepage(void* ptr, size_t size)
702 {
703 #if defined(MADV_HUGEPAGE)
704 (void)madvise(ptr, size, MADV_HUGEPAGE);
705 #elif defined(HAVE_MEMCNTL)
706 struct memcntl_mha m = {.mha_cmd = MHA_MAPSIZE_VA, .mha_pagesize = ZEND_MM_CHUNK_SIZE, .mha_flags = 0};
707 (void)memcntl(ptr, size, MC_HAT_ADVISE, (char *)&m, 0, 0);
708 #elif !defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && !defined(MAP_ALIGNED_SUPER)
709 zend_error_noreturn(E_ERROR, "huge_pages: thp unsupported on this platform");
710 #endif
711 }
712
713 static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment)
714 {
715 void *ptr = zend_mm_mmap(size);
716
717 if (ptr == NULL) {
718 return NULL;
719 } else if (ZEND_MM_ALIGNED_OFFSET(ptr, alignment) == 0) {
720 if (zend_mm_use_huge_pages) {
721 zend_mm_hugepage(ptr, size);
722 }
723 #ifdef __SANITIZE_ADDRESS__
724 ASAN_UNPOISON_MEMORY_REGION(ptr, size);
725 #endif
726 return ptr;
727 } else {
728 size_t offset;
729
730 /* chunk has to be aligned */
731 zend_mm_munmap(ptr, size);
732 ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
733 #ifdef _WIN32
734 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
735 if (offset != 0) {
736 offset = alignment - offset;
737 }
738 zend_mm_munmap(ptr, size + alignment - REAL_PAGE_SIZE);
739 ptr = zend_mm_mmap_fixed((void*)((char*)ptr + offset), size);
740 if (ptr == NULL) { // fix GH-9650, fixed addr range is not free
741 ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
742 if (ptr == NULL) {
743 return NULL;
744 }
745 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
746 if (offset != 0) {
747 ptr = (void*)((char*)ptr + alignment - offset);
748 }
749 }
750 return ptr;
751 #else
752 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
753 if (offset != 0) {
754 offset = alignment - offset;
755 zend_mm_munmap(ptr, offset);
756 ptr = (char*)ptr + offset;
757 alignment -= offset;
758 }
759 if (alignment > REAL_PAGE_SIZE) {
760 zend_mm_munmap((char*)ptr + size, alignment - REAL_PAGE_SIZE);
761 }
762 if (zend_mm_use_huge_pages) {
763 zend_mm_hugepage(ptr, size);
764 }
765 # ifdef __SANITIZE_ADDRESS__
766 ASAN_UNPOISON_MEMORY_REGION(ptr, size);
767 # endif
768 #endif
769 return ptr;
770 }
771 }
772
773 static void *zend_mm_chunk_alloc(zend_mm_heap *heap, size_t size, size_t alignment)
774 {
775 #if ZEND_MM_STORAGE
776 if (UNEXPECTED(heap->storage)) {
777 void *ptr = heap->storage->handlers.chunk_alloc(heap->storage, size, alignment);
778 ZEND_ASSERT(((uintptr_t)((char*)ptr + (alignment-1)) & (alignment-1)) == (uintptr_t)ptr);
779 return ptr;
780 }
781 #endif
782 return zend_mm_chunk_alloc_int(size, alignment);
783 }
784
785 static void zend_mm_chunk_free(zend_mm_heap *heap, void *addr, size_t size)
786 {
787 #if ZEND_MM_STORAGE
788 if (UNEXPECTED(heap->storage)) {
789 heap->storage->handlers.chunk_free(heap->storage, addr, size);
790 return;
791 }
792 #endif
793 zend_mm_munmap(addr, size);
794 }
795
796 static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
797 {
798 #if ZEND_MM_STORAGE
799 if (UNEXPECTED(heap->storage)) {
800 if (heap->storage->handlers.chunk_truncate) {
801 return heap->storage->handlers.chunk_truncate(heap->storage, addr, old_size, new_size);
802 } else {
803 return 0;
804 }
805 }
806 #endif
807 #ifndef _WIN32
808 zend_mm_munmap((char*)addr + new_size, old_size - new_size);
809 return 1;
810 #else
811 return 0;
812 #endif
813 }
814
815 static int zend_mm_chunk_extend(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
816 {
817 #if ZEND_MM_STORAGE
818 if (UNEXPECTED(heap->storage)) {
819 if (heap->storage->handlers.chunk_extend) {
820 return heap->storage->handlers.chunk_extend(heap->storage, addr, old_size, new_size);
821 } else {
822 return 0;
823 }
824 }
825 #endif
826 #ifdef HAVE_MREMAP
827 /* We don't use MREMAP_MAYMOVE due to alignment requirements. */
828 void *ptr = mremap(addr, old_size, new_size, 0);
829 if (ptr == MAP_FAILED) {
830 return 0;
831 }
832 /* Sanity check: The mapping shouldn't have moved. */
833 ZEND_ASSERT(ptr == addr);
834 return 1;
835 #elif !defined(_WIN32)
836 return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL);
837 #else
838 return 0;
839 #endif
840 }
841
842 static zend_always_inline void zend_mm_chunk_init(zend_mm_heap *heap, zend_mm_chunk *chunk)
843 {
844 chunk->heap = heap;
845 chunk->next = heap->main_chunk;
846 chunk->prev = heap->main_chunk->prev;
847 chunk->prev->next = chunk;
848 chunk->next->prev = chunk;
849 /* mark first pages as allocated */
850 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
851 chunk->free_tail = ZEND_MM_FIRST_PAGE;
852 /* the younger chunks have bigger number */
853 chunk->num = chunk->prev->num + 1;
854 /* mark first pages as allocated */
855 chunk->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
856 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
857 }
858
859 /***********************/
860 /* Huge Runs (forward) */
861 /***********************/
862
863 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
864 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
865 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
866
867 #if ZEND_DEBUG
868 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
869 #else
870 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
871 #endif
872
873 /**************/
874 /* Large Runs */
875 /**************/
876
877 #if ZEND_DEBUG
878 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
879 #else
880 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
881 #endif
882 {
883 zend_mm_chunk *chunk = heap->main_chunk;
884 uint32_t page_num, len;
885 int steps = 0;
886
887 while (1) {
888 if (UNEXPECTED(chunk->free_pages < pages_count)) {
889 goto not_found;
890 #if 0
891 } else if (UNEXPECTED(chunk->free_pages + chunk->free_tail == ZEND_MM_PAGES)) {
892 if (UNEXPECTED(ZEND_MM_PAGES - chunk->free_tail < pages_count)) {
893 goto not_found;
894 } else {
895 page_num = chunk->free_tail;
896 goto found;
897 }
898 } else if (0) {
899 /* First-Fit Search */
900 int free_tail = chunk->free_tail;
901 zend_mm_bitset *bitset = chunk->free_map;
902 zend_mm_bitset tmp = *(bitset++);
903 int i = 0;
904
905 while (1) {
906 /* skip allocated blocks */
907 while (tmp == (zend_mm_bitset)-1) {
908 i += ZEND_MM_BITSET_LEN;
909 if (i == ZEND_MM_PAGES) {
910 goto not_found;
911 }
912 tmp = *(bitset++);
913 }
914 /* find first 0 bit */
915 page_num = i + zend_mm_bitset_nts(tmp);
916 /* reset bits from 0 to "bit" */
917 tmp &= tmp + 1;
918 /* skip free blocks */
919 while (tmp == 0) {
920 i += ZEND_MM_BITSET_LEN;
921 len = i - page_num;
922 if (len >= pages_count) {
923 goto found;
924 } else if (i >= free_tail) {
925 goto not_found;
926 }
927 tmp = *(bitset++);
928 }
929 /* find first 1 bit */
930 len = (i + zend_ulong_ntz(tmp)) - page_num;
931 if (len >= pages_count) {
932 goto found;
933 }
934 /* set bits from 0 to "bit" */
935 tmp |= tmp - 1;
936 }
937 #endif
938 } else {
939 /* Best-Fit Search */
940 int best = -1;
941 uint32_t best_len = ZEND_MM_PAGES;
942 uint32_t free_tail = chunk->free_tail;
943 zend_mm_bitset *bitset = chunk->free_map;
944 zend_mm_bitset tmp = *(bitset++);
945 uint32_t i = 0;
946
947 while (1) {
948 /* skip allocated blocks */
949 while (tmp == (zend_mm_bitset)-1) {
950 i += ZEND_MM_BITSET_LEN;
951 if (i == ZEND_MM_PAGES) {
952 if (best > 0) {
953 page_num = best;
954 goto found;
955 } else {
956 goto not_found;
957 }
958 }
959 tmp = *(bitset++);
960 }
961 /* find first 0 bit */
962 page_num = i + zend_mm_bitset_nts(tmp);
963 /* reset bits from 0 to "bit" */
964 tmp &= tmp + 1;
965 /* skip free blocks */
966 while (tmp == 0) {
967 i += ZEND_MM_BITSET_LEN;
968 if (i >= free_tail || i == ZEND_MM_PAGES) {
969 len = ZEND_MM_PAGES - page_num;
970 if (len >= pages_count && len < best_len) {
971 chunk->free_tail = page_num + pages_count;
972 goto found;
973 } else {
974 /* set accurate value */
975 chunk->free_tail = page_num;
976 if (best > 0) {
977 page_num = best;
978 goto found;
979 } else {
980 goto not_found;
981 }
982 }
983 }
984 tmp = *(bitset++);
985 }
986 /* find first 1 bit */
987 len = i + zend_ulong_ntz(tmp) - page_num;
988 if (len >= pages_count) {
989 if (len == pages_count) {
990 goto found;
991 } else if (len < best_len) {
992 best_len = len;
993 best = page_num;
994 }
995 }
996 /* set bits from 0 to "bit" */
997 tmp |= tmp - 1;
998 }
999 }
1000
1001 not_found:
1002 if (chunk->next == heap->main_chunk) {
1003 get_chunk:
1004 if (heap->cached_chunks) {
1005 heap->cached_chunks_count--;
1006 chunk = heap->cached_chunks;
1007 heap->cached_chunks = chunk->next;
1008 } else {
1009 #if ZEND_MM_LIMIT
1010 if (UNEXPECTED(ZEND_MM_CHUNK_SIZE > heap->limit - heap->real_size)) {
1011 if (zend_mm_gc(heap)) {
1012 goto get_chunk;
1013 } else if (heap->overflow == 0) {
1014 #if ZEND_DEBUG
1015 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1016 #else
1017 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, ZEND_MM_PAGE_SIZE * pages_count);
1018 #endif
1019 return NULL;
1020 }
1021 }
1022 #endif
1023 chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1024 if (UNEXPECTED(chunk == NULL)) {
1025 /* insufficient memory */
1026 if (zend_mm_gc(heap) &&
1027 (chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE)) != NULL) {
1028 /* pass */
1029 } else {
1030 #if !ZEND_MM_LIMIT
1031 zend_mm_safe_error(heap, "Out of memory");
1032 #elif ZEND_DEBUG
1033 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1034 #else
1035 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, ZEND_MM_PAGE_SIZE * pages_count);
1036 #endif
1037 return NULL;
1038 }
1039 }
1040 #if ZEND_MM_STAT
1041 do {
1042 size_t size = heap->real_size + ZEND_MM_CHUNK_SIZE;
1043 size_t peak = MAX(heap->real_peak, size);
1044 heap->real_size = size;
1045 heap->real_peak = peak;
1046 } while (0);
1047 #elif ZEND_MM_LIMIT
1048 heap->real_size += ZEND_MM_CHUNK_SIZE;
1049
1050 #endif
1051 }
1052 heap->chunks_count++;
1053 if (heap->chunks_count > heap->peak_chunks_count) {
1054 heap->peak_chunks_count = heap->chunks_count;
1055 }
1056 zend_mm_chunk_init(heap, chunk);
1057 page_num = ZEND_MM_FIRST_PAGE;
1058 len = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1059 goto found;
1060 } else {
1061 chunk = chunk->next;
1062 steps++;
1063 }
1064 }
1065
1066 found:
1067 if (steps > 2 && pages_count < 8) {
1068 ZEND_MM_CHECK(chunk->next->prev == chunk, "zend_mm_heap corrupted");
1069 ZEND_MM_CHECK(chunk->prev->next == chunk, "zend_mm_heap corrupted");
1070
1071 /* move chunk into the head of the linked-list */
1072 chunk->prev->next = chunk->next;
1073 chunk->next->prev = chunk->prev;
1074 chunk->next = heap->main_chunk->next;
1075 chunk->prev = heap->main_chunk;
1076 chunk->prev->next = chunk;
1077 chunk->next->prev = chunk;
1078 }
1079 /* mark run as allocated */
1080 chunk->free_pages -= pages_count;
1081 zend_mm_bitset_set_range(chunk->free_map, page_num, pages_count);
1082 chunk->map[page_num] = ZEND_MM_LRUN(pages_count);
1083 if (page_num == chunk->free_tail) {
1084 chunk->free_tail = page_num + pages_count;
1085 }
1086 return ZEND_MM_PAGE_ADDR(chunk, page_num);
1087 }
1088
1089 static zend_always_inline void *zend_mm_alloc_large_ex(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1090 {
1091 int pages_count = (int)ZEND_MM_SIZE_TO_NUM(size, ZEND_MM_PAGE_SIZE);
1092 #if ZEND_DEBUG
1093 void *ptr = zend_mm_alloc_pages(heap, pages_count, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1094 #else
1095 void *ptr = zend_mm_alloc_pages(heap, pages_count ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1096 #endif
1097 #if ZEND_MM_STAT
1098 do {
1099 size_t size = heap->size + pages_count * ZEND_MM_PAGE_SIZE;
1100 size_t peak = MAX(heap->peak, size);
1101 heap->size = size;
1102 heap->peak = peak;
1103 } while (0);
1104 #endif
1105 return ptr;
1106 }
1107
1108 #if ZEND_DEBUG
1109 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1110 {
1111 return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1112 }
1113 #else
1114 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1115 {
1116 return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1117 }
1118 #endif
1119
1120 static zend_always_inline void zend_mm_delete_chunk(zend_mm_heap *heap, zend_mm_chunk *chunk)
1121 {
1122 ZEND_MM_CHECK(chunk->next->prev == chunk, "zend_mm_heap corrupted");
1123 ZEND_MM_CHECK(chunk->prev->next == chunk, "zend_mm_heap corrupted");
1124
1125 chunk->next->prev = chunk->prev;
1126 chunk->prev->next = chunk->next;
1127 heap->chunks_count--;
1128 if (heap->chunks_count + heap->cached_chunks_count < heap->avg_chunks_count + 0.1
1129 || (heap->chunks_count == heap->last_chunks_delete_boundary
1130 && heap->last_chunks_delete_count >= 4)) {
1131 /* delay deletion */
1132 heap->cached_chunks_count++;
1133 chunk->next = heap->cached_chunks;
1134 heap->cached_chunks = chunk;
1135 } else {
1136 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1137 heap->real_size -= ZEND_MM_CHUNK_SIZE;
1138 #endif
1139 if (!heap->cached_chunks) {
1140 if (heap->chunks_count != heap->last_chunks_delete_boundary) {
1141 heap->last_chunks_delete_boundary = heap->chunks_count;
1142 heap->last_chunks_delete_count = 0;
1143 } else {
1144 heap->last_chunks_delete_count++;
1145 }
1146 }
1147 if (!heap->cached_chunks || chunk->num > heap->cached_chunks->num) {
1148 zend_mm_chunk_free(heap, chunk, ZEND_MM_CHUNK_SIZE);
1149 } else {
1150 //TODO: select the best chunk to delete???
1151 chunk->next = heap->cached_chunks->next;
1152 zend_mm_chunk_free(heap, heap->cached_chunks, ZEND_MM_CHUNK_SIZE);
1153 heap->cached_chunks = chunk;
1154 }
1155 }
1156 }
1157
1158 static zend_always_inline void zend_mm_free_pages_ex(zend_mm_heap *heap, zend_mm_chunk *chunk, uint32_t page_num, uint32_t pages_count, int free_chunk)
1159 {
1160 chunk->free_pages += pages_count;
1161 zend_mm_bitset_reset_range(chunk->free_map, page_num, pages_count);
1162 chunk->map[page_num] = 0;
1163 if (chunk->free_tail == page_num + pages_count) {
1164 /* this setting may be not accurate */
1165 chunk->free_tail = page_num;
1166 }
1167 if (free_chunk && chunk != heap->main_chunk && chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
1168 zend_mm_delete_chunk(heap, chunk);
1169 }
1170 }
1171
1172 static zend_never_inline void zend_mm_free_pages(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1173 {
1174 zend_mm_free_pages_ex(heap, chunk, page_num, pages_count, 1);
1175 }
1176
1177 static zend_always_inline void zend_mm_free_large(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1178 {
1179 #if ZEND_MM_STAT
1180 heap->size -= pages_count * ZEND_MM_PAGE_SIZE;
1181 #endif
1182 zend_mm_free_pages(heap, chunk, page_num, pages_count);
1183 }
1184
1185 /**************/
1186 /* Small Runs */
1187 /**************/
1188
1189 /* higher set bit number (0->N/A, 1->1, 2->2, 4->3, 8->4, 127->7, 128->8 etc) */
1190 static zend_always_inline int zend_mm_small_size_to_bit(int size)
1191 {
1192 #if (defined(__GNUC__) || __has_builtin(__builtin_clz)) && defined(PHP_HAVE_BUILTIN_CLZ)
1193 return (__builtin_clz(size) ^ 0x1f) + 1;
1194 #elif defined(_WIN32)
1195 unsigned long index;
1196
1197 if (!BitScanReverse(&index, (unsigned long)size)) {
1198 /* undefined behavior */
1199 return 64;
1200 }
1201
1202 return (((31 - (int)index) ^ 0x1f) + 1);
1203 #else
1204 int n = 16;
1205 if (size <= 0x00ff) {n -= 8; size = size << 8;}
1206 if (size <= 0x0fff) {n -= 4; size = size << 4;}
1207 if (size <= 0x3fff) {n -= 2; size = size << 2;}
1208 if (size <= 0x7fff) {n -= 1;}
1209 return n;
1210 #endif
1211 }
1212
1213 #ifndef MAX
1214 # define MAX(a, b) (((a) > (b)) ? (a) : (b))
1215 #endif
1216
1217 #ifndef MIN
1218 # define MIN(a, b) (((a) < (b)) ? (a) : (b))
1219 #endif
1220
1221 static zend_always_inline int zend_mm_small_size_to_bin(size_t size)
1222 {
1223 #if 0
1224 int n;
1225 /*0, 1, 2, 3, 4, 5, 6, 7, 8, 9 10, 11, 12*/
1226 static const int f1[] = { 3, 3, 3, 3, 3, 3, 3, 4, 5, 6, 7, 8, 9};
1227 static const int f2[] = { 0, 0, 0, 0, 0, 0, 0, 4, 8, 12, 16, 20, 24};
1228
1229 if (UNEXPECTED(size <= 2)) return 0;
1230 n = zend_mm_small_size_to_bit(size - 1);
1231 return ((size-1) >> f1[n]) + f2[n];
1232 #else
1233 unsigned int t1, t2;
1234
1235 if (size <= 64) {
1236 /* we need to support size == 0 ... */
1237 return (size - !!size) >> 3;
1238 } else {
1239 t1 = size - 1;
1240 t2 = zend_mm_small_size_to_bit(t1) - 3;
1241 t1 = t1 >> t2;
1242 t2 = t2 - 3;
1243 t2 = t2 << 2;
1244 return (int)(t1 + t2);
1245 }
1246 #endif
1247 }
1248
1249 #define ZEND_MM_SMALL_SIZE_TO_BIN(size) zend_mm_small_size_to_bin(size)
1250
1251 static zend_never_inline void *zend_mm_alloc_small_slow(zend_mm_heap *heap, uint32_t bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1252 {
1253 zend_mm_chunk *chunk;
1254 int page_num;
1255 zend_mm_bin *bin;
1256 zend_mm_free_slot *p, *end;
1257
1258 #if ZEND_DEBUG
1259 bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num], bin_data_size[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1260 #else
1261 bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1262 #endif
1263 if (UNEXPECTED(bin == NULL)) {
1264 /* insufficient memory */
1265 return NULL;
1266 }
1267
1268 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(bin, ZEND_MM_CHUNK_SIZE);
1269 page_num = ZEND_MM_ALIGNED_OFFSET(bin, ZEND_MM_CHUNK_SIZE) / ZEND_MM_PAGE_SIZE;
1270 chunk->map[page_num] = ZEND_MM_SRUN(bin_num);
1271 if (bin_pages[bin_num] > 1) {
1272 uint32_t i = 1;
1273
1274 do {
1275 chunk->map[page_num+i] = ZEND_MM_NRUN(bin_num, i);
1276 i++;
1277 } while (i < bin_pages[bin_num]);
1278 }
1279
1280 /* create a linked list of elements from 1 to last */
1281 end = (zend_mm_free_slot*)((char*)bin + (bin_data_size[bin_num] * (bin_elements[bin_num] - 1)));
1282 heap->free_slot[bin_num] = p = (zend_mm_free_slot*)((char*)bin + bin_data_size[bin_num]);
1283 do {
1284 p->next_free_slot = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1285 #if ZEND_DEBUG
1286 do {
1287 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1288 dbg->size = 0;
1289 } while (0);
1290 #endif
1291 p = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1292 } while (p != end);
1293
1294 /* terminate list using NULL */
1295 p->next_free_slot = NULL;
1296 #if ZEND_DEBUG
1297 do {
1298 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1299 dbg->size = 0;
1300 } while (0);
1301 #endif
1302
1303 /* return first element */
1304 return bin;
1305 }
1306
1307 static zend_always_inline void *zend_mm_alloc_small(zend_mm_heap *heap, int bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1308 {
1309 #if ZEND_MM_STAT
1310 do {
1311 size_t size = heap->size + bin_data_size[bin_num];
1312 size_t peak = MAX(heap->peak, size);
1313 heap->size = size;
1314 heap->peak = peak;
1315 } while (0);
1316 #endif
1317
1318 if (EXPECTED(heap->free_slot[bin_num] != NULL)) {
1319 zend_mm_free_slot *p = heap->free_slot[bin_num];
1320 heap->free_slot[bin_num] = p->next_free_slot;
1321 return p;
1322 } else {
1323 return zend_mm_alloc_small_slow(heap, bin_num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1324 }
1325 }
1326
1327 static zend_always_inline void zend_mm_free_small(zend_mm_heap *heap, void *ptr, int bin_num)
1328 {
1329 zend_mm_free_slot *p;
1330
1331 #if ZEND_MM_STAT
1332 heap->size -= bin_data_size[bin_num];
1333 #endif
1334
1335 #if ZEND_DEBUG
1336 do {
1337 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1338 dbg->size = 0;
1339 } while (0);
1340 #endif
1341
1342 p = (zend_mm_free_slot*)ptr;
1343 p->next_free_slot = heap->free_slot[bin_num];
1344 heap->free_slot[bin_num] = p;
1345 }
1346
1347 /********/
1348 /* Heap */
1349 /********/
1350
1351 #if ZEND_DEBUG
1352 static zend_always_inline zend_mm_debug_info *zend_mm_get_debug_info(zend_mm_heap *heap, void *ptr)
1353 {
1354 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1355 zend_mm_chunk *chunk;
1356 int page_num;
1357 zend_mm_page_info info;
1358
1359 ZEND_MM_CHECK(page_offset != 0, "zend_mm_heap corrupted");
1360 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1361 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1362 info = chunk->map[page_num];
1363 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1364 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1365 int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1366 return (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1367 } else /* if (info & ZEND_MM_IS_LRUN) */ {
1368 int pages_count = ZEND_MM_LRUN_PAGES(info);
1369
1370 return (zend_mm_debug_info*)((char*)ptr + ZEND_MM_PAGE_SIZE * pages_count - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1371 }
1372 }
1373 #endif
1374
1375 static zend_always_inline void *zend_mm_alloc_heap(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1376 {
1377 void *ptr;
1378 #if ZEND_DEBUG
1379 size_t real_size = size;
1380 zend_mm_debug_info *dbg;
1381
1382 /* special handling for zero-size allocation */
1383 size = MAX(size, 1);
1384 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1385 if (UNEXPECTED(size < real_size)) {
1386 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", ZEND_MM_ALIGNED_SIZE(real_size), ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1387 return NULL;
1388 }
1389 #endif
1390 if (EXPECTED(size <= ZEND_MM_MAX_SMALL_SIZE)) {
1391 ptr = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1392 #if ZEND_DEBUG
1393 dbg = zend_mm_get_debug_info(heap, ptr);
1394 dbg->size = real_size;
1395 dbg->filename = __zend_filename;
1396 dbg->orig_filename = __zend_orig_filename;
1397 dbg->lineno = __zend_lineno;
1398 dbg->orig_lineno = __zend_orig_lineno;
1399 #endif
1400 return ptr;
1401 } else if (EXPECTED(size <= ZEND_MM_MAX_LARGE_SIZE)) {
1402 ptr = zend_mm_alloc_large(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1403 #if ZEND_DEBUG
1404 dbg = zend_mm_get_debug_info(heap, ptr);
1405 dbg->size = real_size;
1406 dbg->filename = __zend_filename;
1407 dbg->orig_filename = __zend_orig_filename;
1408 dbg->lineno = __zend_lineno;
1409 dbg->orig_lineno = __zend_orig_lineno;
1410 #endif
1411 return ptr;
1412 } else {
1413 #if ZEND_DEBUG
1414 size = real_size;
1415 #endif
1416 return zend_mm_alloc_huge(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1417 }
1418 }
1419
1420 static zend_always_inline void zend_mm_free_heap(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1421 {
1422 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1423
1424 if (UNEXPECTED(page_offset == 0)) {
1425 if (ptr != NULL) {
1426 zend_mm_free_huge(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1427 }
1428 } else {
1429 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1430 int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1431 zend_mm_page_info info = chunk->map[page_num];
1432
1433 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1434 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1435 zend_mm_free_small(heap, ptr, ZEND_MM_SRUN_BIN_NUM(info));
1436 } else /* if (info & ZEND_MM_IS_LRUN) */ {
1437 int pages_count = ZEND_MM_LRUN_PAGES(info);
1438
1439 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1440 zend_mm_free_large(heap, chunk, page_num, pages_count);
1441 }
1442 }
1443 }
1444
1445 static size_t zend_mm_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1446 {
1447 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1448
1449 if (UNEXPECTED(page_offset == 0)) {
1450 return zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1451 } else {
1452 zend_mm_chunk *chunk;
1453 #if 0 && ZEND_DEBUG
1454 zend_mm_debug_info *dbg = zend_mm_get_debug_info(heap, ptr);
1455 return dbg->size;
1456 #else
1457 int page_num;
1458 zend_mm_page_info info;
1459
1460 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1461 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1462 info = chunk->map[page_num];
1463 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1464 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1465 return bin_data_size[ZEND_MM_SRUN_BIN_NUM(info)];
1466 } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1467 return ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1468 }
1469 #endif
1470 }
1471 }
1472
1473 static zend_never_inline void *zend_mm_realloc_slow(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1474 {
1475 void *ret;
1476
1477 #if ZEND_MM_STAT
1478 do {
1479 size_t orig_peak = heap->peak;
1480 #endif
1481 ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1482 memcpy(ret, ptr, copy_size);
1483 zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1484 #if ZEND_MM_STAT
1485 heap->peak = MAX(orig_peak, heap->size);
1486 } while (0);
1487 #endif
1488 return ret;
1489 }
1490
1491 static zend_never_inline void *zend_mm_realloc_huge(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1492 {
1493 size_t old_size;
1494 size_t new_size;
1495 #if ZEND_DEBUG
1496 size_t real_size;
1497 #endif
1498
1499 old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1500 #if ZEND_DEBUG
1501 real_size = size;
1502 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1503 #endif
1504 if (size > ZEND_MM_MAX_LARGE_SIZE) {
1505 #if ZEND_DEBUG
1506 size = real_size;
1507 #endif
1508 #ifdef ZEND_WIN32
1509 /* On Windows we don't have ability to extend huge blocks in-place.
1510 * We allocate them with 2MB size granularity, to avoid many
1511 * reallocations when they are extended by small pieces
1512 */
1513 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE));
1514 #else
1515 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE);
1516 #endif
1517 if (new_size == old_size) {
1518 #if ZEND_DEBUG
1519 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1520 #else
1521 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1522 #endif
1523 return ptr;
1524 } else if (new_size < old_size) {
1525 /* unmup tail */
1526 if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) {
1527 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1528 heap->real_size -= old_size - new_size;
1529 #endif
1530 #if ZEND_MM_STAT
1531 heap->size -= old_size - new_size;
1532 #endif
1533 #if ZEND_DEBUG
1534 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1535 #else
1536 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1537 #endif
1538 return ptr;
1539 }
1540 } else /* if (new_size > old_size) */ {
1541 #if ZEND_MM_LIMIT
1542 if (UNEXPECTED(new_size - old_size > heap->limit - heap->real_size)) {
1543 if (zend_mm_gc(heap) && new_size - old_size <= heap->limit - heap->real_size) {
1544 /* pass */
1545 } else if (heap->overflow == 0) {
1546 #if ZEND_DEBUG
1547 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1548 #else
1549 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1550 #endif
1551 return NULL;
1552 }
1553 }
1554 #endif
1555 /* try to map tail right after this block */
1556 if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) {
1557 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1558 heap->real_size += new_size - old_size;
1559 #endif
1560 #if ZEND_MM_STAT
1561 heap->real_peak = MAX(heap->real_peak, heap->real_size);
1562 heap->size += new_size - old_size;
1563 heap->peak = MAX(heap->peak, heap->size);
1564 #endif
1565 #if ZEND_DEBUG
1566 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1567 #else
1568 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1569 #endif
1570 return ptr;
1571 }
1572 }
1573 }
1574
1575 return zend_mm_realloc_slow(heap, ptr, size, MIN(old_size, copy_size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1576 }
1577
1578 static zend_always_inline void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, bool use_copy_size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1579 {
1580 size_t page_offset;
1581 size_t old_size;
1582 size_t new_size;
1583 void *ret;
1584 #if ZEND_DEBUG
1585 zend_mm_debug_info *dbg;
1586 #endif
1587
1588 page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1589 if (UNEXPECTED(page_offset == 0)) {
1590 if (EXPECTED(ptr == NULL)) {
1591 return _zend_mm_alloc(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1592 } else {
1593 return zend_mm_realloc_huge(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1594 }
1595 } else {
1596 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1597 int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1598 zend_mm_page_info info = chunk->map[page_num];
1599 #if ZEND_DEBUG
1600 size_t real_size = size;
1601
1602 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1603 #endif
1604
1605 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1606 if (info & ZEND_MM_IS_SRUN) {
1607 int old_bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1608
1609 do {
1610 old_size = bin_data_size[old_bin_num];
1611
1612 /* Check if requested size fits into current bin */
1613 if (size <= old_size) {
1614 /* Check if truncation is necessary */
1615 if (old_bin_num > 0 && size < bin_data_size[old_bin_num - 1]) {
1616 /* truncation */
1617 ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1618 copy_size = use_copy_size ? MIN(size, copy_size) : size;
1619 memcpy(ret, ptr, copy_size);
1620 zend_mm_free_small(heap, ptr, old_bin_num);
1621 } else {
1622 /* reallocation in-place */
1623 ret = ptr;
1624 }
1625 } else if (size <= ZEND_MM_MAX_SMALL_SIZE) {
1626 /* small extension */
1627
1628 #if ZEND_MM_STAT
1629 do {
1630 size_t orig_peak = heap->peak;
1631 #endif
1632 ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1633 copy_size = use_copy_size ? MIN(old_size, copy_size) : old_size;
1634 memcpy(ret, ptr, copy_size);
1635 zend_mm_free_small(heap, ptr, old_bin_num);
1636 #if ZEND_MM_STAT
1637 heap->peak = MAX(orig_peak, heap->size);
1638 } while (0);
1639 #endif
1640 } else {
1641 /* slow reallocation */
1642 break;
1643 }
1644
1645 #if ZEND_DEBUG
1646 dbg = zend_mm_get_debug_info(heap, ret);
1647 dbg->size = real_size;
1648 dbg->filename = __zend_filename;
1649 dbg->orig_filename = __zend_orig_filename;
1650 dbg->lineno = __zend_lineno;
1651 dbg->orig_lineno = __zend_orig_lineno;
1652 #endif
1653 return ret;
1654 } while (0);
1655
1656 } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1657 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1658 old_size = ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1659 if (size > ZEND_MM_MAX_SMALL_SIZE && size <= ZEND_MM_MAX_LARGE_SIZE) {
1660 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE);
1661 if (new_size == old_size) {
1662 #if ZEND_DEBUG
1663 dbg = zend_mm_get_debug_info(heap, ptr);
1664 dbg->size = real_size;
1665 dbg->filename = __zend_filename;
1666 dbg->orig_filename = __zend_orig_filename;
1667 dbg->lineno = __zend_lineno;
1668 dbg->orig_lineno = __zend_orig_lineno;
1669 #endif
1670 return ptr;
1671 } else if (new_size < old_size) {
1672 /* free tail pages */
1673 int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1674 int rest_pages_count = (int)((old_size - new_size) / ZEND_MM_PAGE_SIZE);
1675
1676 #if ZEND_MM_STAT
1677 heap->size -= rest_pages_count * ZEND_MM_PAGE_SIZE;
1678 #endif
1679 chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1680 chunk->free_pages += rest_pages_count;
1681 zend_mm_bitset_reset_range(chunk->free_map, page_num + new_pages_count, rest_pages_count);
1682 #if ZEND_DEBUG
1683 dbg = zend_mm_get_debug_info(heap, ptr);
1684 dbg->size = real_size;
1685 dbg->filename = __zend_filename;
1686 dbg->orig_filename = __zend_orig_filename;
1687 dbg->lineno = __zend_lineno;
1688 dbg->orig_lineno = __zend_orig_lineno;
1689 #endif
1690 return ptr;
1691 } else /* if (new_size > old_size) */ {
1692 int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1693 int old_pages_count = (int)(old_size / ZEND_MM_PAGE_SIZE);
1694
1695 /* try to allocate tail pages after this block */
1696 if (page_num + new_pages_count <= ZEND_MM_PAGES &&
1697 zend_mm_bitset_is_free_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count)) {
1698 #if ZEND_MM_STAT
1699 do {
1700 size_t size = heap->size + (new_size - old_size);
1701 size_t peak = MAX(heap->peak, size);
1702 heap->size = size;
1703 heap->peak = peak;
1704 } while (0);
1705 #endif
1706 chunk->free_pages -= new_pages_count - old_pages_count;
1707 zend_mm_bitset_set_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count);
1708 chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1709 #if ZEND_DEBUG
1710 dbg = zend_mm_get_debug_info(heap, ptr);
1711 dbg->size = real_size;
1712 dbg->filename = __zend_filename;
1713 dbg->orig_filename = __zend_orig_filename;
1714 dbg->lineno = __zend_lineno;
1715 dbg->orig_lineno = __zend_orig_lineno;
1716 #endif
1717 return ptr;
1718 }
1719 }
1720 }
1721 }
1722 #if ZEND_DEBUG
1723 size = real_size;
1724 #endif
1725 }
1726
1727 copy_size = MIN(old_size, copy_size);
1728 return zend_mm_realloc_slow(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1729 }
1730
1731 /*********************/
1732 /* Huge Runs (again) */
1733 /*********************/
1734
1735 #if ZEND_DEBUG
1736 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1737 #else
1738 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1739 #endif
1740 {
1741 zend_mm_huge_list *list = (zend_mm_huge_list*)zend_mm_alloc_heap(heap, sizeof(zend_mm_huge_list) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1742 list->ptr = ptr;
1743 list->size = size;
1744 list->next = heap->huge_list;
1745 #if ZEND_DEBUG
1746 list->dbg.size = dbg_size;
1747 list->dbg.filename = __zend_filename;
1748 list->dbg.orig_filename = __zend_orig_filename;
1749 list->dbg.lineno = __zend_lineno;
1750 list->dbg.orig_lineno = __zend_orig_lineno;
1751 #endif
1752 heap->huge_list = list;
1753 }
1754
1755 static size_t zend_mm_del_huge_block(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1756 {
1757 zend_mm_huge_list *prev = NULL;
1758 zend_mm_huge_list *list = heap->huge_list;
1759 while (list != NULL) {
1760 if (list->ptr == ptr) {
1761 size_t size;
1762
1763 if (prev) {
1764 prev->next = list->next;
1765 } else {
1766 heap->huge_list = list->next;
1767 }
1768 size = list->size;
1769 zend_mm_free_heap(heap, list ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1770 return size;
1771 }
1772 prev = list;
1773 list = list->next;
1774 }
1775 ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1776 return 0;
1777 }
1778
1779 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1780 {
1781 zend_mm_huge_list *list = heap->huge_list;
1782 while (list != NULL) {
1783 if (list->ptr == ptr) {
1784 return list->size;
1785 }
1786 list = list->next;
1787 }
1788 ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1789 return 0;
1790 }
1791
1792 #if ZEND_DEBUG
1793 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1794 #else
1795 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1796 #endif
1797 {
1798 zend_mm_huge_list *list = heap->huge_list;
1799 while (list != NULL) {
1800 if (list->ptr == ptr) {
1801 list->size = size;
1802 #if ZEND_DEBUG
1803 list->dbg.size = dbg_size;
1804 list->dbg.filename = __zend_filename;
1805 list->dbg.orig_filename = __zend_orig_filename;
1806 list->dbg.lineno = __zend_lineno;
1807 list->dbg.orig_lineno = __zend_orig_lineno;
1808 #endif
1809 return;
1810 }
1811 list = list->next;
1812 }
1813 }
1814
1815 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1816 {
1817 #ifdef ZEND_WIN32
1818 /* On Windows we don't have ability to extend huge blocks in-place.
1819 * We allocate them with 2MB size granularity, to avoid many
1820 * reallocations when they are extended by small pieces
1821 */
1822 size_t alignment = MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE);
1823 #else
1824 size_t alignment = REAL_PAGE_SIZE;
1825 #endif
1826 size_t new_size = ZEND_MM_ALIGNED_SIZE_EX(size, alignment);
1827 void *ptr;
1828
1829 if (UNEXPECTED(new_size < size)) {
1830 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", size, alignment);
1831 }
1832
1833 #if ZEND_MM_LIMIT
1834 if (UNEXPECTED(new_size > heap->limit - heap->real_size)) {
1835 if (zend_mm_gc(heap) && new_size <= heap->limit - heap->real_size) {
1836 /* pass */
1837 } else if (heap->overflow == 0) {
1838 #if ZEND_DEBUG
1839 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1840 #else
1841 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1842 #endif
1843 return NULL;
1844 }
1845 }
1846 #endif
1847 ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE);
1848 if (UNEXPECTED(ptr == NULL)) {
1849 /* insufficient memory */
1850 if (zend_mm_gc(heap) &&
1851 (ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE)) != NULL) {
1852 /* pass */
1853 } else {
1854 #if !ZEND_MM_LIMIT
1855 zend_mm_safe_error(heap, "Out of memory");
1856 #elif ZEND_DEBUG
1857 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1858 #else
1859 zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, size);
1860 #endif
1861 return NULL;
1862 }
1863 }
1864 #if ZEND_DEBUG
1865 zend_mm_add_huge_block(heap, ptr, new_size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1866 #else
1867 zend_mm_add_huge_block(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1868 #endif
1869 #if ZEND_MM_STAT
1870 do {
1871 size_t size = heap->real_size + new_size;
1872 size_t peak = MAX(heap->real_peak, size);
1873 heap->real_size = size;
1874 heap->real_peak = peak;
1875 } while (0);
1876 do {
1877 size_t size = heap->size + new_size;
1878 size_t peak = MAX(heap->peak, size);
1879 heap->size = size;
1880 heap->peak = peak;
1881 } while (0);
1882 #elif ZEND_MM_LIMIT
1883 heap->real_size += new_size;
1884 #endif
1885 return ptr;
1886 }
1887
1888 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1889 {
1890 size_t size;
1891
1892 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE) == 0, "zend_mm_heap corrupted");
1893 size = zend_mm_del_huge_block(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1894 zend_mm_chunk_free(heap, ptr, size);
1895 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1896 heap->real_size -= size;
1897 #endif
1898 #if ZEND_MM_STAT
1899 heap->size -= size;
1900 #endif
1901 }
1902
1903 /******************/
1904 /* Initialization */
1905 /******************/
1906
1907 static zend_mm_heap *zend_mm_init(void)
1908 {
1909 zend_mm_chunk *chunk = (zend_mm_chunk*)zend_mm_chunk_alloc_int(ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1910 zend_mm_heap *heap;
1911
1912 if (UNEXPECTED(chunk == NULL)) {
1913 #if ZEND_MM_ERROR
1914 fprintf(stderr, "Can't initialize heap\n");
1915 #endif
1916 return NULL;
1917 }
1918 heap = &chunk->heap_slot;
1919 chunk->heap = heap;
1920 chunk->next = chunk;
1921 chunk->prev = chunk;
1922 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1923 chunk->free_tail = ZEND_MM_FIRST_PAGE;
1924 chunk->num = 0;
1925 chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
1926 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
1927 heap->main_chunk = chunk;
1928 heap->cached_chunks = NULL;
1929 heap->chunks_count = 1;
1930 heap->peak_chunks_count = 1;
1931 heap->cached_chunks_count = 0;
1932 heap->avg_chunks_count = 1.0;
1933 heap->last_chunks_delete_boundary = 0;
1934 heap->last_chunks_delete_count = 0;
1935 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1936 heap->real_size = ZEND_MM_CHUNK_SIZE;
1937 #endif
1938 #if ZEND_MM_STAT
1939 heap->real_peak = ZEND_MM_CHUNK_SIZE;
1940 heap->size = 0;
1941 heap->peak = 0;
1942 #endif
1943 #if ZEND_MM_LIMIT
1944 heap->limit = (size_t)Z_L(-1) >> 1;
1945 heap->overflow = 0;
1946 #endif
1947 #if ZEND_MM_CUSTOM
1948 heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
1949 #endif
1950 #if ZEND_MM_STORAGE
1951 heap->storage = NULL;
1952 #endif
1953 heap->huge_list = NULL;
1954 return heap;
1955 }
1956
1957 ZEND_API size_t zend_mm_gc(zend_mm_heap *heap)
1958 {
1959 zend_mm_free_slot *p, **q;
1960 zend_mm_chunk *chunk;
1961 size_t page_offset;
1962 int page_num;
1963 zend_mm_page_info info;
1964 uint32_t i, free_counter;
1965 bool has_free_pages;
1966 size_t collected = 0;
1967
1968 #if ZEND_MM_CUSTOM
1969 if (heap->use_custom_heap) {
1970 return 0;
1971 }
1972 #endif
1973
1974 for (i = 0; i < ZEND_MM_BINS; i++) {
1975 has_free_pages = false;
1976 p = heap->free_slot[i];
1977 while (p != NULL) {
1978 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
1979 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1980 page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
1981 ZEND_ASSERT(page_offset != 0);
1982 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1983 info = chunk->map[page_num];
1984 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1985 if (info & ZEND_MM_IS_LRUN) {
1986 page_num -= ZEND_MM_NRUN_OFFSET(info);
1987 info = chunk->map[page_num];
1988 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1989 ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
1990 }
1991 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
1992 free_counter = ZEND_MM_SRUN_FREE_COUNTER(info) + 1;
1993 if (free_counter == bin_elements[i]) {
1994 has_free_pages = true;
1995 }
1996 chunk->map[page_num] = ZEND_MM_SRUN_EX(i, free_counter);
1997 p = p->next_free_slot;
1998 }
1999
2000 if (!has_free_pages) {
2001 continue;
2002 }
2003
2004 q = &heap->free_slot[i];
2005 p = *q;
2006 while (p != NULL) {
2007 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
2008 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
2009 page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
2010 ZEND_ASSERT(page_offset != 0);
2011 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
2012 info = chunk->map[page_num];
2013 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
2014 if (info & ZEND_MM_IS_LRUN) {
2015 page_num -= ZEND_MM_NRUN_OFFSET(info);
2016 info = chunk->map[page_num];
2017 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
2018 ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
2019 }
2020 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
2021 if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[i]) {
2022 /* remove from cache */
2023 p = p->next_free_slot;
2024 *q = p;
2025 } else {
2026 q = &p->next_free_slot;
2027 p = *q;
2028 }
2029 }
2030 }
2031
2032 chunk = heap->main_chunk;
2033 do {
2034 i = ZEND_MM_FIRST_PAGE;
2035 while (i < chunk->free_tail) {
2036 if (zend_mm_bitset_is_set(chunk->free_map, i)) {
2037 info = chunk->map[i];
2038 if (info & ZEND_MM_IS_SRUN) {
2039 int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
2040 int pages_count = bin_pages[bin_num];
2041
2042 if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[bin_num]) {
2043 /* all elements are free */
2044 zend_mm_free_pages_ex(heap, chunk, i, pages_count, 0);
2045 collected += pages_count;
2046 } else {
2047 /* reset counter */
2048 chunk->map[i] = ZEND_MM_SRUN(bin_num);
2049 }
2050 i += bin_pages[bin_num];
2051 } else /* if (info & ZEND_MM_IS_LRUN) */ {
2052 i += ZEND_MM_LRUN_PAGES(info);
2053 }
2054 } else {
2055 i++;
2056 }
2057 }
2058 if (chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
2059 zend_mm_chunk *next_chunk = chunk->next;
2060
2061 zend_mm_delete_chunk(heap, chunk);
2062 chunk = next_chunk;
2063 } else {
2064 chunk = chunk->next;
2065 }
2066 } while (chunk != heap->main_chunk);
2067
2068 return collected * ZEND_MM_PAGE_SIZE;
2069 }
2070
2071 #if ZEND_DEBUG
2072 /******************/
2073 /* Leak detection */
2074 /******************/
2075
2076 static zend_long zend_mm_find_leaks_small(zend_mm_chunk *p, uint32_t i, uint32_t j, zend_leak_info *leak)
2077 {
2078 bool empty = true;
2079 zend_long count = 0;
2080 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2081 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * (j + 1) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2082
2083 while (j < bin_elements[bin_num]) {
2084 if (dbg->size != 0) {
2085 if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2086 count++;
2087 dbg->size = 0;
2088 dbg->filename = NULL;
2089 dbg->lineno = 0;
2090 } else {
2091 empty = false;
2092 }
2093 }
2094 j++;
2095 dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2096 }
2097 if (empty) {
2098 zend_mm_bitset_reset_range(p->free_map, i, bin_pages[bin_num]);
2099 }
2100 return count;
2101 }
2102
2103 static zend_long zend_mm_find_leaks(zend_mm_heap *heap, zend_mm_chunk *p, uint32_t i, zend_leak_info *leak)
2104 {
2105 zend_long count = 0;
2106
2107 do {
2108 while (i < p->free_tail) {
2109 if (zend_mm_bitset_is_set(p->free_map, i)) {
2110 if (p->map[i] & ZEND_MM_IS_SRUN) {
2111 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2112 count += zend_mm_find_leaks_small(p, i, 0, leak);
2113 i += bin_pages[bin_num];
2114 } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2115 int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2116 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2117
2118 if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2119 count++;
2120 }
2121 zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2122 i += pages_count;
2123 }
2124 } else {
2125 i++;
2126 }
2127 }
2128 p = p->next;
2129 i = ZEND_MM_FIRST_PAGE;
2130 } while (p != heap->main_chunk);
2131 return count;
2132 }
2133
2134 static zend_long zend_mm_find_leaks_huge(zend_mm_heap *heap, zend_mm_huge_list *list)
2135 {
2136 zend_long count = 0;
2137 zend_mm_huge_list *prev = list;
2138 zend_mm_huge_list *p = list->next;
2139
2140 while (p) {
2141 if (p->dbg.filename == list->dbg.filename && p->dbg.lineno == list->dbg.lineno) {
2142 prev->next = p->next;
2143 zend_mm_chunk_free(heap, p->ptr, p->size);
2144 zend_mm_free_heap(heap, p, NULL, 0, NULL, 0);
2145 count++;
2146 } else {
2147 prev = p;
2148 }
2149 p = prev->next;
2150 }
2151
2152 return count;
2153 }
2154
2155 static void zend_mm_check_leaks(zend_mm_heap *heap)
2156 {
2157 zend_mm_huge_list *list;
2158 zend_mm_chunk *p;
2159 zend_leak_info leak;
2160 zend_long repeated = 0;
2161 uint32_t total = 0;
2162 uint32_t i, j;
2163
2164 /* find leaked huge blocks and free them */
2165 list = heap->huge_list;
2166 while (list) {
2167 zend_mm_huge_list *q = list;
2168
2169 leak.addr = list->ptr;
2170 leak.size = list->dbg.size;
2171 leak.filename = list->dbg.filename;
2172 leak.orig_filename = list->dbg.orig_filename;
2173 leak.lineno = list->dbg.lineno;
2174 leak.orig_lineno = list->dbg.orig_lineno;
2175
2176 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2177 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2178 repeated = zend_mm_find_leaks_huge(heap, list);
2179 total += 1 + repeated;
2180 if (repeated) {
2181 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated);
2182 }
2183
2184 heap->huge_list = list = list->next;
2185 zend_mm_chunk_free(heap, q->ptr, q->size);
2186 zend_mm_free_heap(heap, q, NULL, 0, NULL, 0);
2187 }
2188
2189 /* for each chunk */
2190 p = heap->main_chunk;
2191 do {
2192 i = ZEND_MM_FIRST_PAGE;
2193 while (i < p->free_tail) {
2194 if (zend_mm_bitset_is_set(p->free_map, i)) {
2195 if (p->map[i] & ZEND_MM_IS_SRUN) {
2196 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2197 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2198
2199 j = 0;
2200 while (j < bin_elements[bin_num]) {
2201 if (dbg->size != 0) {
2202 leak.addr = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * j);
2203 leak.size = dbg->size;
2204 leak.filename = dbg->filename;
2205 leak.orig_filename = dbg->orig_filename;
2206 leak.lineno = dbg->lineno;
2207 leak.orig_lineno = dbg->orig_lineno;
2208
2209 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2210 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2211
2212 dbg->size = 0;
2213 dbg->filename = NULL;
2214 dbg->lineno = 0;
2215
2216 repeated = zend_mm_find_leaks_small(p, i, j + 1, &leak) +
2217 zend_mm_find_leaks(heap, p, i + bin_pages[bin_num], &leak);
2218 total += 1 + repeated;
2219 if (repeated) {
2220 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated);
2221 }
2222 }
2223 dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2224 j++;
2225 }
2226 i += bin_pages[bin_num];
2227 } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2228 int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2229 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2230
2231 leak.addr = (void*)((char*)p + ZEND_MM_PAGE_SIZE * i);
2232 leak.size = dbg->size;
2233 leak.filename = dbg->filename;
2234 leak.orig_filename = dbg->orig_filename;
2235 leak.lineno = dbg->lineno;
2236 leak.orig_lineno = dbg->orig_lineno;
2237
2238 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2239 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2240
2241 zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2242
2243 repeated = zend_mm_find_leaks(heap, p, i + pages_count, &leak);
2244 total += 1 + repeated;
2245 if (repeated) {
2246 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated);
2247 }
2248 i += pages_count;
2249 }
2250 } else {
2251 i++;
2252 }
2253 }
2254 p = p->next;
2255 } while (p != heap->main_chunk);
2256 if (total) {
2257 zend_message_dispatcher(ZMSG_MEMORY_LEAKS_GRAND_TOTAL, &total);
2258 }
2259 }
2260 #endif
2261
2262 #if ZEND_MM_CUSTOM
2263 static void *tracked_malloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
2264 static void tracked_free_all(void);
2265 #endif
2266
2267 void zend_mm_shutdown(zend_mm_heap *heap, bool full, bool silent)
2268 {
2269 zend_mm_chunk *p;
2270 zend_mm_huge_list *list;
2271
2272 #if ZEND_MM_CUSTOM
2273 if (heap->use_custom_heap) {
2274 if (heap->custom_heap._malloc == tracked_malloc) {
2275 if (silent) {
2276 tracked_free_all();
2277 }
2278 zend_hash_clean(heap->tracked_allocs);
2279 if (full) {
2280 zend_hash_destroy(heap->tracked_allocs);
2281 free(heap->tracked_allocs);
2282 /* Make sure the heap free below does not use tracked_free(). */
2283 heap->custom_heap._free = __zend_free;
2284 }
2285 heap->size = 0;
2286 }
2287
2288 if (full) {
2289 heap->custom_heap._free(heap ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC);
2290 }
2291 return;
2292 }
2293 #endif
2294
2295 #if ZEND_DEBUG
2296 if (!silent) {
2297 char *tmp = getenv("ZEND_ALLOC_PRINT_LEAKS");
2298 if (!tmp || ZEND_ATOL(tmp)) {
2299 zend_mm_check_leaks(heap);
2300 }
2301 }
2302 #endif
2303
2304 /* free huge blocks */
2305 list = heap->huge_list;
2306 heap->huge_list = NULL;
2307 while (list) {
2308 zend_mm_huge_list *q = list;
2309 list = list->next;
2310 zend_mm_chunk_free(heap, q->ptr, q->size);
2311 }
2312
2313 /* move all chunks except of the first one into the cache */
2314 p = heap->main_chunk->next;
2315 while (p != heap->main_chunk) {
2316 zend_mm_chunk *q = p->next;
2317 p->next = heap->cached_chunks;
2318 heap->cached_chunks = p;
2319 p = q;
2320 heap->chunks_count--;
2321 heap->cached_chunks_count++;
2322 }
2323
2324 if (full) {
2325 /* free all cached chunks */
2326 while (heap->cached_chunks) {
2327 p = heap->cached_chunks;
2328 heap->cached_chunks = p->next;
2329 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2330 }
2331 /* free the first chunk */
2332 zend_mm_chunk_free(heap, heap->main_chunk, ZEND_MM_CHUNK_SIZE);
2333 } else {
2334 /* free some cached chunks to keep average count */
2335 heap->avg_chunks_count = (heap->avg_chunks_count + (double)heap->peak_chunks_count) / 2.0;
2336 while ((double)heap->cached_chunks_count + 0.9 > heap->avg_chunks_count &&
2337 heap->cached_chunks) {
2338 p = heap->cached_chunks;
2339 heap->cached_chunks = p->next;
2340 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2341 heap->cached_chunks_count--;
2342 }
2343 /* clear cached chunks */
2344 p = heap->cached_chunks;
2345 while (p != NULL) {
2346 zend_mm_chunk *q = p->next;
2347 memset(p, 0, sizeof(zend_mm_chunk));
2348 p->next = q;
2349 p = q;
2350 }
2351
2352 /* reinitialize the first chunk and heap */
2353 p = heap->main_chunk;
2354 p->heap = &p->heap_slot;
2355 p->next = p;
2356 p->prev = p;
2357 p->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
2358 p->free_tail = ZEND_MM_FIRST_PAGE;
2359 p->num = 0;
2360
2361 #if ZEND_MM_STAT
2362 heap->size = heap->peak = 0;
2363 #endif
2364 memset(heap->free_slot, 0, sizeof(heap->free_slot));
2365 #if ZEND_MM_STAT || ZEND_MM_LIMIT
2366 heap->real_size = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE;
2367 #endif
2368 #if ZEND_MM_STAT
2369 heap->real_peak = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE;
2370 #endif
2371 heap->chunks_count = 1;
2372 heap->peak_chunks_count = 1;
2373 heap->last_chunks_delete_boundary = 0;
2374 heap->last_chunks_delete_count = 0;
2375
2376 memset(p->free_map, 0, sizeof(p->free_map) + sizeof(p->map));
2377 p->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
2378 p->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
2379 }
2380 }
2381
2382 /**************/
2383 /* PUBLIC API */
2384 /**************/
2385
2386 ZEND_API void* ZEND_FASTCALL _zend_mm_alloc(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2387 {
2388 return zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2389 }
2390
2391 ZEND_API void ZEND_FASTCALL _zend_mm_free(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2392 {
2393 zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2394 }
2395
2396 void* ZEND_FASTCALL _zend_mm_realloc(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2397 {
2398 return zend_mm_realloc_heap(heap, ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2399 }
2400
2401 void* ZEND_FASTCALL _zend_mm_realloc2(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2402 {
2403 return zend_mm_realloc_heap(heap, ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2404 }
2405
2406 ZEND_API size_t ZEND_FASTCALL _zend_mm_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2407 {
2408 #if ZEND_MM_CUSTOM
2409 if (UNEXPECTED(heap->use_custom_heap)) {
2410 if (heap->custom_heap._malloc == tracked_malloc) {
2411 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2412 zval *size_zv = zend_hash_index_find(heap->tracked_allocs, h);
2413 if (size_zv) {
2414 return Z_LVAL_P(size_zv);
2415 }
2416 }
2417 return 0;
2418 }
2419 #endif
2420 return zend_mm_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2421 }
2422
2423 /**********************/
2424 /* Allocation Manager */
2425 /**********************/
2426
2427 typedef struct _zend_alloc_globals {
2428 zend_mm_heap *mm_heap;
2429 } zend_alloc_globals;
2430
2431 #ifdef ZTS
2432 static int alloc_globals_id;
2433 static size_t alloc_globals_offset;
2434 # define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v)
2435 #else
2436 # define AG(v) (alloc_globals.v)
2437 static zend_alloc_globals alloc_globals;
2438 #endif
2439
2440 ZEND_API bool is_zend_mm(void)
2441 {
2442 #if ZEND_MM_CUSTOM
2443 return !AG(mm_heap)->use_custom_heap;
2444 #else
2445 return 1;
2446 #endif
2447 }
2448
2449 ZEND_API bool is_zend_ptr(const void *ptr)
2450 {
2451 #if ZEND_MM_CUSTOM
2452 if (AG(mm_heap)->use_custom_heap) {
2453 if (AG(mm_heap)->custom_heap._malloc == tracked_malloc) {
2454 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2455 zval *size_zv = zend_hash_index_find(AG(mm_heap)->tracked_allocs, h);
2456 if (size_zv) {
2457 return 1;
2458 }
2459 }
2460 return 0;
2461 }
2462 #endif
2463
2464 if (AG(mm_heap)->main_chunk) {
2465 zend_mm_chunk *chunk = AG(mm_heap)->main_chunk;
2466
2467 do {
2468 if (ptr >= (void*)chunk
2469 && ptr < (void*)((char*)chunk + ZEND_MM_CHUNK_SIZE)) {
2470 return 1;
2471 }
2472 chunk = chunk->next;
2473 } while (chunk != AG(mm_heap)->main_chunk);
2474 }
2475
2476 if (AG(mm_heap)->huge_list) {
2477 zend_mm_huge_list *block = AG(mm_heap)->huge_list;
2478
2479 do {
2480 if (ptr >= (void*)block
2481 && ptr < (void*)((char*)block + block->size)) {
2482 return 1;
2483 }
2484 block = block->next;
2485 } while (block != AG(mm_heap)->huge_list);
2486 }
2487 return 0;
2488 }
2489
2490 #if !ZEND_DEBUG && defined(HAVE_BUILTIN_CONSTANT_P)
2491 #undef _emalloc
2492
2493 #if ZEND_MM_CUSTOM
2494 # define ZEND_MM_CUSTOM_ALLOCATOR(size) do { \
2495 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2496 return AG(mm_heap)->custom_heap._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2497 } \
2498 } while (0)
2499 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr) do { \
2500 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2501 AG(mm_heap)->custom_heap._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2502 return; \
2503 } \
2504 } while (0)
2505 #else
2506 # define ZEND_MM_CUSTOM_ALLOCATOR(size)
2507 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr)
2508 #endif
2509
2510 # define _ZEND_BIN_ALLOCATOR(_num, _size, _elements, _pages, x, y) \
2511 ZEND_API void* ZEND_FASTCALL _emalloc_ ## _size(void) { \
2512 ZEND_MM_CUSTOM_ALLOCATOR(_size); \
2513 return zend_mm_alloc_small(AG(mm_heap), _num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2514 }
2515
2516 ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR, x, y)
2517
2518 ZEND_API void* ZEND_FASTCALL _emalloc_large(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2519 {
2520 ZEND_MM_CUSTOM_ALLOCATOR(size);
2521 return zend_mm_alloc_large_ex(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2522 }
2523
2524 ZEND_API void* ZEND_FASTCALL _emalloc_huge(size_t size)
2525 {
2526 ZEND_MM_CUSTOM_ALLOCATOR(size);
2527 return zend_mm_alloc_huge(AG(mm_heap), size);
2528 }
2529
2530 #if ZEND_DEBUG
2531 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2532 ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2533 ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2534 { \
2535 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); \
2536 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2537 int page_num = page_offset / ZEND_MM_PAGE_SIZE; \
2538 ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2539 ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_SRUN); \
2540 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(chunk->map[page_num]) == _num); \
2541 zend_mm_free_small(AG(mm_heap), ptr, _num); \
2542 } \
2543 }
2544 #else
2545 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2546 ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2547 ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2548 { \
2549 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2550 ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2551 zend_mm_free_small(AG(mm_heap), ptr, _num); \
2552 } \
2553 }
2554 #endif
2555
2556 ZEND_MM_BINS_INFO(_ZEND_BIN_FREE, x, y)
2557
2558 ZEND_API void ZEND_FASTCALL _efree_large(void *ptr, size_t size)
2559 {
2560 ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2561 {
2562 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
2563 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
2564 int page_num = page_offset / ZEND_MM_PAGE_SIZE;
2565 uint32_t pages_count = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE) / ZEND_MM_PAGE_SIZE;
2566
2567 ZEND_MM_CHECK(chunk->heap == AG(mm_heap) && ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
2568 ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_LRUN);
2569 ZEND_ASSERT(ZEND_MM_LRUN_PAGES(chunk->map[page_num]) == pages_count);
2570 zend_mm_free_large(AG(mm_heap), chunk, page_num, pages_count);
2571 }
2572 }
2573
2574 ZEND_API void ZEND_FASTCALL _efree_huge(void *ptr, size_t size)
2575 {
2576
2577 ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2578 zend_mm_free_huge(AG(mm_heap), ptr);
2579 }
2580 #endif
2581
2582 ZEND_API void* ZEND_FASTCALL _emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2583 {
2584 #if ZEND_MM_CUSTOM
2585 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2586 return AG(mm_heap)->custom_heap._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2587 }
2588 #endif
2589 return zend_mm_alloc_heap(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2590 }
2591
2592 ZEND_API void ZEND_FASTCALL _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2593 {
2594 #if ZEND_MM_CUSTOM
2595 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2596 AG(mm_heap)->custom_heap._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2597 return;
2598 }
2599 #endif
2600 zend_mm_free_heap(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2601 }
2602
2603 ZEND_API void* ZEND_FASTCALL _erealloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2604 {
2605 #if ZEND_MM_CUSTOM
2606 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2607 return AG(mm_heap)->custom_heap._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2608 }
2609 #endif
2610 return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2611 }
2612
2613 ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2614 {
2615 #if ZEND_MM_CUSTOM
2616 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2617 return AG(mm_heap)->custom_heap._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2618 }
2619 #endif
2620 return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2621 }
2622
2623 ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2624 {
2625 return _zend_mm_block_size(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2626 }
2627
2628 ZEND_API void* ZEND_FASTCALL _safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2629 {
2630 return _emalloc(zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2631 }
2632
2633 ZEND_API void* ZEND_FASTCALL _safe_malloc(size_t nmemb, size_t size, size_t offset)
2634 {
2635 return pemalloc(zend_safe_address_guarded(nmemb, size, offset), 1);
2636 }
2637
2638 ZEND_API void* ZEND_FASTCALL _safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2639 {
2640 return _erealloc(ptr, zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2641 }
2642
2643 ZEND_API void* ZEND_FASTCALL _safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset)
2644 {
2645 return perealloc(ptr, zend_safe_address_guarded(nmemb, size, offset), 1);
2646 }
2647
2648 ZEND_API void* ZEND_FASTCALL _ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2649 {
2650 void *p;
2651
2652 size = zend_safe_address_guarded(nmemb, size, 0);
2653 p = _emalloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2654 memset(p, 0, size);
2655 return p;
2656 }
2657
2658 ZEND_API char* ZEND_FASTCALL _estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2659 {
2660 size_t length;
2661 char *p;
2662
2663 length = strlen(s);
2664 if (UNEXPECTED(length + 1 == 0)) {
2665 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2666 }
2667 p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2668 memcpy(p, s, length+1);
2669 return p;
2670 }
2671
2672 ZEND_API char* ZEND_FASTCALL _estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2673 {
2674 char *p;
2675
2676 if (UNEXPECTED(length + 1 == 0)) {
2677 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2678 }
2679 p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2680 memcpy(p, s, length);
2681 p[length] = 0;
2682 return p;
2683 }
2684
2685 static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void);
2686
2687 ZEND_API char* ZEND_FASTCALL zend_strndup(const char *s, size_t length)
2688 {
2689 char *p;
2690
2691 if (UNEXPECTED(length + 1 == 0)) {
2692 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2693 }
2694 p = (char *) malloc(length + 1);
2695 if (UNEXPECTED(p == NULL)) {
2696 zend_out_of_memory();
2697 }
2698 if (EXPECTED(length)) {
2699 memcpy(p, s, length);
2700 }
2701 p[length] = 0;
2702 return p;
2703 }
2704
2705 ZEND_API zend_result zend_set_memory_limit(size_t memory_limit)
2706 {
2707 #if ZEND_MM_LIMIT
2708 zend_mm_heap *heap = AG(mm_heap);
2709
2710 if (UNEXPECTED(memory_limit < heap->real_size)) {
2711 if (memory_limit >= heap->real_size - heap->cached_chunks_count * ZEND_MM_CHUNK_SIZE) {
2712 /* free some cached chunks to fit into new memory limit */
2713 do {
2714 zend_mm_chunk *p = heap->cached_chunks;
2715 heap->cached_chunks = p->next;
2716 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2717 heap->cached_chunks_count--;
2718 heap->real_size -= ZEND_MM_CHUNK_SIZE;
2719 } while (memory_limit < heap->real_size);
2720 return SUCCESS;
2721 }
2722 return FAILURE;
2723 }
2724 AG(mm_heap)->limit = memory_limit;
2725 #endif
2726 return SUCCESS;
2727 }
2728
2729 ZEND_API bool zend_alloc_in_memory_limit_error_reporting(void)
2730 {
2731 #if ZEND_MM_LIMIT
2732 return AG(mm_heap)->overflow;
2733 #else
2734 return false;
2735 #endif
2736 }
2737
2738 ZEND_API size_t zend_memory_usage(bool real_usage)
2739 {
2740 #if ZEND_MM_STAT
2741 if (real_usage) {
2742 return AG(mm_heap)->real_size;
2743 } else {
2744 size_t usage = AG(mm_heap)->size;
2745 return usage;
2746 }
2747 #endif
2748 return 0;
2749 }
2750
2751 ZEND_API size_t zend_memory_peak_usage(bool real_usage)
2752 {
2753 #if ZEND_MM_STAT
2754 if (real_usage) {
2755 return AG(mm_heap)->real_peak;
2756 } else {
2757 return AG(mm_heap)->peak;
2758 }
2759 #endif
2760 return 0;
2761 }
2762
2763 ZEND_API void zend_memory_reset_peak_usage(void)
2764 {
2765 #if ZEND_MM_STAT
2766 AG(mm_heap)->real_peak = AG(mm_heap)->real_size;
2767 AG(mm_heap)->peak = AG(mm_heap)->size;
2768 #endif
2769 }
2770
2771 ZEND_API void shutdown_memory_manager(bool silent, bool full_shutdown)
2772 {
2773 zend_mm_shutdown(AG(mm_heap), full_shutdown, silent);
2774 }
2775
2776 static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void)
2777 {
2778 fprintf(stderr, "Out of memory\n");
2779 exit(1);
2780 }
2781
2782 #if ZEND_MM_CUSTOM
2783 static zend_always_inline void tracked_add(zend_mm_heap *heap, void *ptr, size_t size) {
2784 zval size_zv;
2785 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2786 ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr);
2787 ZVAL_LONG(&size_zv, size);
2788 zend_hash_index_add_new(heap->tracked_allocs, h, &size_zv);
2789 }
2790
2791 static zend_always_inline zval *tracked_get_size_zv(zend_mm_heap *heap, void *ptr) {
2792 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2793 zval *size_zv = zend_hash_index_find(heap->tracked_allocs, h);
2794 ZEND_ASSERT(size_zv && "Trying to free pointer not allocated through ZendMM");
2795 return size_zv;
2796 }
2797
2798 static zend_always_inline void tracked_check_limit(zend_mm_heap *heap, size_t add_size) {
2799 if (add_size > heap->limit - heap->size && !heap->overflow) {
2800 #if ZEND_DEBUG
2801 zend_mm_safe_error(heap,
2802 "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)",
2803 heap->limit, "file", 0, add_size);
2804 #else
2805 zend_mm_safe_error(heap,
2806 "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)",
2807 heap->limit, add_size);
2808 #endif
2809 }
2810 }
2811
2812 static void *tracked_malloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2813 {
2814 zend_mm_heap *heap = AG(mm_heap);
2815 tracked_check_limit(heap, size);
2816
2817 void *ptr = malloc(size);
2818 if (!ptr) {
2819 zend_out_of_memory();
2820 }
2821
2822 tracked_add(heap, ptr, size);
2823 heap->size += size;
2824 return ptr;
2825 }
2826
2827 static void tracked_free(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) {
2828 if (!ptr) {
2829 return;
2830 }
2831
2832 zend_mm_heap *heap = AG(mm_heap);
2833 zval *size_zv = tracked_get_size_zv(heap, ptr);
2834 heap->size -= Z_LVAL_P(size_zv);
2835 zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) size_zv);
2836 free(ptr);
2837 }
2838
2839 static void *tracked_realloc(void *ptr, size_t new_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) {
2840 zend_mm_heap *heap = AG(mm_heap);
2841 zval *old_size_zv = NULL;
2842 size_t old_size = 0;
2843 if (ptr) {
2844 old_size_zv = tracked_get_size_zv(heap, ptr);
2845 old_size = Z_LVAL_P(old_size_zv);
2846 }
2847
2848 if (new_size > old_size) {
2849 tracked_check_limit(heap, new_size - old_size);
2850 }
2851
2852 /* Delete information about old allocation only after checking the memory limit. */
2853 if (old_size_zv) {
2854 zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) old_size_zv);
2855 }
2856
2857 ptr = __zend_realloc(ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2858 tracked_add(heap, ptr, new_size);
2859 heap->size += new_size - old_size;
2860 return ptr;
2861 }
2862
2863 static void tracked_free_all(void) {
2864 HashTable *tracked_allocs = AG(mm_heap)->tracked_allocs;
2865 zend_ulong h;
2866 ZEND_HASH_FOREACH_NUM_KEY(tracked_allocs, h) {
2867 void *ptr = (void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2);
2868 free(ptr);
2869 } ZEND_HASH_FOREACH_END();
2870 }
2871 #endif
2872
2873 static void alloc_globals_ctor(zend_alloc_globals *alloc_globals)
2874 {
2875 char *tmp;
2876
2877 #if ZEND_MM_CUSTOM
2878 tmp = getenv("USE_ZEND_ALLOC");
2879 if (tmp && !ZEND_ATOL(tmp)) {
2880 bool tracked = (tmp = getenv("USE_TRACKED_ALLOC")) && ZEND_ATOL(tmp);
2881 zend_mm_heap *mm_heap = alloc_globals->mm_heap = malloc(sizeof(zend_mm_heap));
2882 memset(mm_heap, 0, sizeof(zend_mm_heap));
2883 mm_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2884 mm_heap->limit = (size_t)Z_L(-1) >> 1;
2885 mm_heap->overflow = 0;
2886
2887 if (!tracked) {
2888 /* Use system allocator. */
2889 mm_heap->custom_heap._malloc = __zend_malloc;
2890 mm_heap->custom_heap._free = __zend_free;
2891 mm_heap->custom_heap._realloc = __zend_realloc;
2892 } else {
2893 /* Use system allocator and track allocations for auto-free. */
2894 mm_heap->custom_heap._malloc = tracked_malloc;
2895 mm_heap->custom_heap._free = tracked_free;
2896 mm_heap->custom_heap._realloc = tracked_realloc;
2897 mm_heap->tracked_allocs = malloc(sizeof(HashTable));
2898 zend_hash_init(mm_heap->tracked_allocs, 1024, NULL, NULL, 1);
2899 }
2900 return;
2901 }
2902 #endif
2903
2904 tmp = getenv("USE_ZEND_ALLOC_HUGE_PAGES");
2905 if (tmp && ZEND_ATOL(tmp)) {
2906 zend_mm_use_huge_pages = true;
2907 }
2908 alloc_globals->mm_heap = zend_mm_init();
2909 }
2910
2911 #ifdef ZTS
2912 static void alloc_globals_dtor(zend_alloc_globals *alloc_globals)
2913 {
2914 zend_mm_shutdown(alloc_globals->mm_heap, 1, 1);
2915 }
2916 #endif
2917
2918 ZEND_API void start_memory_manager(void)
2919 {
2920 #ifdef ZTS
2921 ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
2922 #else
2923 alloc_globals_ctor(&alloc_globals);
2924 #endif
2925 #ifndef _WIN32
2926 # if defined(_SC_PAGESIZE)
2927 REAL_PAGE_SIZE = sysconf(_SC_PAGESIZE);
2928 # elif defined(_SC_PAGE_SIZE)
2929 REAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
2930 # endif
2931 #endif
2932 }
2933
2934 ZEND_API zend_mm_heap *zend_mm_set_heap(zend_mm_heap *new_heap)
2935 {
2936 zend_mm_heap *old_heap;
2937
2938 old_heap = AG(mm_heap);
2939 AG(mm_heap) = (zend_mm_heap*)new_heap;
2940 return (zend_mm_heap*)old_heap;
2941 }
2942
2943 ZEND_API zend_mm_heap *zend_mm_get_heap(void)
2944 {
2945 return AG(mm_heap);
2946 }
2947
2948 ZEND_API bool zend_mm_is_custom_heap(zend_mm_heap *new_heap)
2949 {
2950 #if ZEND_MM_CUSTOM
2951 return AG(mm_heap)->use_custom_heap;
2952 #else
2953 return 0;
2954 #endif
2955 }
2956
2957 ZEND_API void zend_mm_set_custom_handlers(zend_mm_heap *heap,
2958 void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2959 void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2960 void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC))
2961 {
2962 #if ZEND_MM_CUSTOM
2963 zend_mm_heap *_heap = (zend_mm_heap*)heap;
2964
2965 if (!_malloc && !_free && !_realloc) {
2966 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
2967 } else {
2968 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2969 _heap->custom_heap._malloc = _malloc;
2970 _heap->custom_heap._free = _free;
2971 _heap->custom_heap._realloc = _realloc;
2972 }
2973 #endif
2974 }
2975
2976 ZEND_API void zend_mm_get_custom_handlers(zend_mm_heap *heap,
2977 void* (**_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2978 void (**_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2979 void* (**_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC))
2980 {
2981 #if ZEND_MM_CUSTOM
2982 zend_mm_heap *_heap = (zend_mm_heap*)heap;
2983
2984 if (heap->use_custom_heap) {
2985 *_malloc = _heap->custom_heap._malloc;
2986 *_free = _heap->custom_heap._free;
2987 *_realloc = _heap->custom_heap._realloc;
2988 } else {
2989 *_malloc = NULL;
2990 *_free = NULL;
2991 *_realloc = NULL;
2992 }
2993 #else
2994 *_malloc = NULL;
2995 *_free = NULL;
2996 *_realloc = NULL;
2997 #endif
2998 }
2999
3000 ZEND_API zend_mm_storage *zend_mm_get_storage(zend_mm_heap *heap)
3001 {
3002 #if ZEND_MM_STORAGE
3003 return heap->storage;
3004 #else
3005 return NULL
3006 #endif
3007 }
3008
3009 ZEND_API zend_mm_heap *zend_mm_startup(void)
3010 {
3011 return zend_mm_init();
3012 }
3013
3014 ZEND_API zend_mm_heap *zend_mm_startup_ex(const zend_mm_handlers *handlers, void *data, size_t data_size)
3015 {
3016 #if ZEND_MM_STORAGE
3017 zend_mm_storage tmp_storage, *storage;
3018 zend_mm_chunk *chunk;
3019 zend_mm_heap *heap;
3020
3021 memcpy((zend_mm_handlers*)&tmp_storage.handlers, handlers, sizeof(zend_mm_handlers));
3022 tmp_storage.data = data;
3023 chunk = (zend_mm_chunk*)handlers->chunk_alloc(&tmp_storage, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
3024 if (UNEXPECTED(chunk == NULL)) {
3025 #if ZEND_MM_ERROR
3026 fprintf(stderr, "Can't initialize heap\n");
3027 #endif
3028 return NULL;
3029 }
3030 heap = &chunk->heap_slot;
3031 chunk->heap = heap;
3032 chunk->next = chunk;
3033 chunk->prev = chunk;
3034 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
3035 chunk->free_tail = ZEND_MM_FIRST_PAGE;
3036 chunk->num = 0;
3037 chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
3038 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
3039 heap->main_chunk = chunk;
3040 heap->cached_chunks = NULL;
3041 heap->chunks_count = 1;
3042 heap->peak_chunks_count = 1;
3043 heap->cached_chunks_count = 0;
3044 heap->avg_chunks_count = 1.0;
3045 heap->last_chunks_delete_boundary = 0;
3046 heap->last_chunks_delete_count = 0;
3047 #if ZEND_MM_STAT || ZEND_MM_LIMIT
3048 heap->real_size = ZEND_MM_CHUNK_SIZE;
3049 #endif
3050 #if ZEND_MM_STAT
3051 heap->real_peak = ZEND_MM_CHUNK_SIZE;
3052 heap->size = 0;
3053 heap->peak = 0;
3054 #endif
3055 #if ZEND_MM_LIMIT
3056 heap->limit = (size_t)Z_L(-1) >> 1;
3057 heap->overflow = 0;
3058 #endif
3059 #if ZEND_MM_CUSTOM
3060 heap->use_custom_heap = 0;
3061 #endif
3062 heap->storage = &tmp_storage;
3063 heap->huge_list = NULL;
3064 memset(heap->free_slot, 0, sizeof(heap->free_slot));
3065 storage = _zend_mm_alloc(heap, sizeof(zend_mm_storage) + data_size ZEND_FILE_LINE_CC ZEND_FILE_LINE_CC);
3066 if (!storage) {
3067 handlers->chunk_free(&tmp_storage, chunk, ZEND_MM_CHUNK_SIZE);
3068 #if ZEND_MM_ERROR
3069 fprintf(stderr, "Can't initialize heap\n");
3070 #endif
3071 return NULL;
3072 }
3073 memcpy(storage, &tmp_storage, sizeof(zend_mm_storage));
3074 if (data) {
3075 storage->data = (void*)(((char*)storage + sizeof(zend_mm_storage)));
3076 memcpy(storage->data, data, data_size);
3077 }
3078 heap->storage = storage;
3079 return heap;
3080 #else
3081 return NULL;
3082 #endif
3083 }
3084
3085 ZEND_API void * __zend_malloc(size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
3086 {
3087 void *tmp = malloc(len);
3088 if (EXPECTED(tmp || !len)) {
3089 return tmp;
3090 }
3091 zend_out_of_memory();
3092 }
3093
3094 ZEND_API void * __zend_calloc(size_t nmemb, size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
3095 {
3096 void *tmp;
3097
3098 len = zend_safe_address_guarded(nmemb, len, 0);
3099 tmp = __zend_malloc(len ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
3100 memset(tmp, 0, len);
3101 return tmp;
3102 }
3103
3104 ZEND_API void * __zend_realloc(void *p, size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
3105 {
3106 p = realloc(p, len);
3107 if (EXPECTED(p || !len)) {
3108 return p;
3109 }
3110 zend_out_of_memory();
3111 }
3112
3113 ZEND_API void __zend_free(void *p ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
3114 {
3115 free(p);
3116 return;
3117 }
3118
3119 ZEND_API char * __zend_strdup(const char *s)
3120 {
3121 char *tmp = strdup(s);
3122 if (EXPECTED(tmp)) {
3123 return tmp;
3124 }
3125 zend_out_of_memory();
3126 }
3127
3128 #ifdef ZTS
3129 size_t zend_mm_globals_size(void)
3130 {
3131 return sizeof(zend_alloc_globals);
3132 }
3133 #endif
3134
