xref: /PHP-8.2/Zend/zend_alloc.c (revision 1ff277de)
1 /*
2    +----------------------------------------------------------------------+
3    | Zend Engine                                                          |
4    +----------------------------------------------------------------------+
5    | Copyright (c) Zend Technologies Ltd. (http://www.zend.com)           |
6    +----------------------------------------------------------------------+
7    | This source file is subject to version 2.00 of the Zend license,     |
8    | that is bundled with this package in the file LICENSE, and is        |
9    | available through the world-wide-web at the following url:           |
10    | http://www.zend.com/license/2_00.txt.                                |
11    | If you did not receive a copy of the Zend license and are unable to  |
12    | obtain it through the world-wide-web, please send a note to          |
13    | license@zend.com so we can mail you a copy immediately.              |
14    +----------------------------------------------------------------------+
15    | Authors: Andi Gutmans <andi@php.net>                                 |
16    |          Zeev Suraski <zeev@php.net>                                 |
17    |          Dmitry Stogov <dmitry@php.net>                              |
18    +----------------------------------------------------------------------+
19 */
20 
21 /*
22  * zend_alloc is designed to be a modern CPU cache friendly memory manager
23  * for PHP. Most ideas are taken from jemalloc and tcmalloc implementations.
24  *
25  * All allocations are split into 3 categories:
26  *
27  * Huge  - the size is greater than CHUNK size (~2M by default), allocation is
28  *         performed using mmap(). The result is aligned on 2M boundary.
29  *
30  * Large - a number of 4096K pages inside a CHUNK. Large blocks
31  *         are always aligned on page boundary.
32  *
33  * Small - less than 3/4 of page size. Small sizes are rounded up to nearest
34  *         greater predefined small size (there are 30 predefined sizes:
35  *         8, 16, 24, 32, ... 3072). Small blocks are allocated from
36  *         RUNs. Each RUN is allocated as a single or few following pages.
37  *         Allocation inside RUNs implemented using linked list of free
38  *         elements. The result is aligned to 8 bytes.
39  *
40  * zend_alloc allocates memory from OS by CHUNKs, these CHUNKs and huge memory
41  * blocks are always aligned to CHUNK boundary. So it's very easy to determine
42  * the CHUNK owning the certain pointer. Regular CHUNKs reserve a single
43  * page at start for special purpose. It contains bitset of free pages,
44  * few bitset for available runs of predefined small sizes, map of pages that
45  * keeps information about usage of each page in this CHUNK, etc.
46  *
47  * zend_alloc provides familiar emalloc/efree/erealloc API, but in addition it
48  * provides specialized and optimized routines to allocate blocks of predefined
49  * sizes (e.g. emalloc_2(), emallc_4(), ..., emalloc_large(), etc)
50  * The library uses C preprocessor tricks that substitute calls to emalloc()
51  * with more specialized routines when the requested size is known.
52  */
53 
54 #include "zend.h"
55 #include "zend_alloc.h"
56 #include "zend_globals.h"
57 #include "zend_operators.h"
58 #include "zend_multiply.h"
59 #include "zend_bitset.h"
60 #include "zend_mmap.h"
61 #include <signal.h>
62 
63 #ifdef HAVE_UNISTD_H
64 # include <unistd.h>
65 #endif
66 
67 #ifdef ZEND_WIN32
68 # include <wincrypt.h>
69 # include <process.h>
70 # include "win32/winutil.h"
71 #endif
72 
73 #include <stdio.h>
74 #include <stdlib.h>
75 #include <string.h>
76 
77 #include <sys/types.h>
78 #include <sys/stat.h>
79 #include <limits.h>
80 #include <fcntl.h>
81 #include <errno.h>
82 
83 #ifndef _WIN32
84 # include <sys/mman.h>
85 # ifndef MAP_ANON
86 #  ifdef MAP_ANONYMOUS
87 #   define MAP_ANON MAP_ANONYMOUS
88 #  endif
89 # endif
90 # ifndef MAP_FAILED
91 #  define MAP_FAILED ((void*)-1)
92 # endif
93 # ifndef MAP_POPULATE
94 #  define MAP_POPULATE 0
95 # endif
96 #  if defined(_SC_PAGESIZE) || (_SC_PAGE_SIZE)
97 #    define REAL_PAGE_SIZE _real_page_size
98 static size_t _real_page_size = ZEND_MM_PAGE_SIZE;
99 #  endif
100 # ifdef MAP_ALIGNED_SUPER
101 #    define MAP_HUGETLB MAP_ALIGNED_SUPER
102 # endif
103 #endif
104 
105 #ifndef REAL_PAGE_SIZE
106 # define REAL_PAGE_SIZE ZEND_MM_PAGE_SIZE
107 #endif
108 
109 /* NetBSD has an mremap() function with a signature that is incompatible with Linux (WTF?),
110  * so pretend it doesn't exist. */
111 #ifndef __linux__
112 # undef HAVE_MREMAP
113 #endif
114 
115 #ifndef __APPLE__
116 # define ZEND_MM_FD -1
117 #else
118 # include <mach/vm_statistics.h>
119 /* Mac allows to track anonymous page via vmmap per TAG id.
120  * user land applications are allowed to take from 240 to 255.
121  */
122 # define ZEND_MM_FD VM_MAKE_TAG(250U)
123 #endif
124 
125 #ifndef ZEND_MM_STAT
126 # define ZEND_MM_STAT 1    /* track current and peak memory usage            */
127 #endif
128 #ifndef ZEND_MM_LIMIT
129 # define ZEND_MM_LIMIT 1   /* support for user-defined memory limit          */
130 #endif
131 #ifndef ZEND_MM_CUSTOM
132 # define ZEND_MM_CUSTOM 1  /* support for custom memory allocator            */
133                            /* USE_ZEND_ALLOC=0 may switch to system malloc() */
134 #endif
135 #ifndef ZEND_MM_STORAGE
136 # define ZEND_MM_STORAGE 1 /* support for custom memory storage              */
137 #endif
138 #ifndef ZEND_MM_ERROR
139 # define ZEND_MM_ERROR 1   /* report system errors                           */
140 #endif
141 
142 #ifndef ZEND_MM_CHECK
143 # define ZEND_MM_CHECK(condition, message)  do { \
144 		if (UNEXPECTED(!(condition))) { \
145 			zend_mm_panic(message); \
146 		} \
147 	} while (0)
148 #endif
149 
150 typedef uint32_t   zend_mm_page_info; /* 4-byte integer */
151 typedef zend_ulong zend_mm_bitset;    /* 4-byte or 8-byte integer */
152 
153 #define ZEND_MM_ALIGNED_OFFSET(size, alignment) \
154 	(((size_t)(size)) & ((alignment) - 1))
155 #define ZEND_MM_ALIGNED_BASE(size, alignment) \
156 	(((size_t)(size)) & ~((alignment) - 1))
157 #define ZEND_MM_SIZE_TO_NUM(size, alignment) \
158 	(((size_t)(size) + ((alignment) - 1)) / (alignment))
159 
160 #define ZEND_MM_BITSET_LEN		(sizeof(zend_mm_bitset) * 8)       /* 32 or 64 */
161 #define ZEND_MM_PAGE_MAP_LEN	(ZEND_MM_PAGES / ZEND_MM_BITSET_LEN) /* 16 or 8 */
162 
163 typedef zend_mm_bitset zend_mm_page_map[ZEND_MM_PAGE_MAP_LEN];     /* 64B */
164 
165 #define ZEND_MM_IS_FRUN                  0x00000000
166 #define ZEND_MM_IS_LRUN                  0x40000000
167 #define ZEND_MM_IS_SRUN                  0x80000000
168 
169 #define ZEND_MM_LRUN_PAGES_MASK          0x000003ff
170 #define ZEND_MM_LRUN_PAGES_OFFSET        0
171 
172 #define ZEND_MM_SRUN_BIN_NUM_MASK        0x0000001f
173 #define ZEND_MM_SRUN_BIN_NUM_OFFSET      0
174 
175 #define ZEND_MM_SRUN_FREE_COUNTER_MASK   0x01ff0000
176 #define ZEND_MM_SRUN_FREE_COUNTER_OFFSET 16
177 
178 #define ZEND_MM_NRUN_OFFSET_MASK         0x01ff0000
179 #define ZEND_MM_NRUN_OFFSET_OFFSET       16
180 
181 #define ZEND_MM_LRUN_PAGES(info)         (((info) & ZEND_MM_LRUN_PAGES_MASK) >> ZEND_MM_LRUN_PAGES_OFFSET)
182 #define ZEND_MM_SRUN_BIN_NUM(info)       (((info) & ZEND_MM_SRUN_BIN_NUM_MASK) >> ZEND_MM_SRUN_BIN_NUM_OFFSET)
183 #define ZEND_MM_SRUN_FREE_COUNTER(info)  (((info) & ZEND_MM_SRUN_FREE_COUNTER_MASK) >> ZEND_MM_SRUN_FREE_COUNTER_OFFSET)
184 #define ZEND_MM_NRUN_OFFSET(info)        (((info) & ZEND_MM_NRUN_OFFSET_MASK) >> ZEND_MM_NRUN_OFFSET_OFFSET)
185 
186 #define ZEND_MM_FRUN()                   ZEND_MM_IS_FRUN
187 #define ZEND_MM_LRUN(count)              (ZEND_MM_IS_LRUN | ((count) << ZEND_MM_LRUN_PAGES_OFFSET))
188 #define ZEND_MM_SRUN(bin_num)            (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET))
189 #define ZEND_MM_SRUN_EX(bin_num, count)  (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((count) << ZEND_MM_SRUN_FREE_COUNTER_OFFSET))
190 #define ZEND_MM_NRUN(bin_num, offset)    (ZEND_MM_IS_SRUN | ZEND_MM_IS_LRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((offset) << ZEND_MM_NRUN_OFFSET_OFFSET))
191 
192 #define ZEND_MM_BINS 30
193 
194 typedef struct  _zend_mm_page      zend_mm_page;
195 typedef struct  _zend_mm_bin       zend_mm_bin;
196 typedef struct  _zend_mm_free_slot zend_mm_free_slot;
197 typedef struct  _zend_mm_chunk     zend_mm_chunk;
198 typedef struct  _zend_mm_huge_list zend_mm_huge_list;
199 
200 static bool zend_mm_use_huge_pages = false;
201 
202 /*
203  * Memory is retrieved from OS by chunks of fixed size 2MB.
204  * Inside chunk it's managed by pages of fixed size 4096B.
205  * So each chunk consists from 512 pages.
206  * The first page of each chunk is reserved for chunk header.
207  * It contains service information about all pages.
208  *
209  * free_pages - current number of free pages in this chunk
210  *
211  * free_tail  - number of continuous free pages at the end of chunk
212  *
213  * free_map   - bitset (a bit for each page). The bit is set if the corresponding
214  *              page is allocated. Allocator for "large sizes" may easily find a
215  *              free page (or a continuous number of pages) searching for zero
216  *              bits.
217  *
218  * map        - contains service information for each page. (32-bits for each
219  *              page).
220  *    usage:
221  *				(2 bits)
222  * 				FRUN - free page,
223  *              LRUN - first page of "large" allocation
224  *              SRUN - first page of a bin used for "small" allocation
225  *
226  *    lrun_pages:
227  *              (10 bits) number of allocated pages
228  *
229  *    srun_bin_num:
230  *              (5 bits) bin number (e.g. 0 for sizes 0-2, 1 for 3-4,
231  *               2 for 5-8, 3 for 9-16 etc) see zend_alloc_sizes.h
232  */
233 
234 struct _zend_mm_heap {
235 #if ZEND_MM_CUSTOM
236 	int                use_custom_heap;
237 #endif
238 #if ZEND_MM_STORAGE
239 	zend_mm_storage   *storage;
240 #endif
241 #if ZEND_MM_STAT
242 	size_t             size;                    /* current memory usage */
243 	size_t             peak;                    /* peak memory usage */
244 #endif
245 	zend_mm_free_slot *free_slot[ZEND_MM_BINS]; /* free lists for small sizes */
246 #if ZEND_MM_STAT || ZEND_MM_LIMIT
247 	size_t             real_size;               /* current size of allocated pages */
248 #endif
249 #if ZEND_MM_STAT
250 	size_t             real_peak;               /* peak size of allocated pages */
251 #endif
252 #if ZEND_MM_LIMIT
253 	size_t             limit;                   /* memory limit */
254 	int                overflow;                /* memory overflow flag */
255 #endif
256 
257 	zend_mm_huge_list *huge_list;               /* list of huge allocated blocks */
258 
259 	zend_mm_chunk     *main_chunk;
260 	zend_mm_chunk     *cached_chunks;			/* list of unused chunks */
261 	int                chunks_count;			/* number of allocated chunks */
262 	int                peak_chunks_count;		/* peak number of allocated chunks for current request */
263 	int                cached_chunks_count;		/* number of cached chunks */
264 	double             avg_chunks_count;		/* average number of chunks allocated per request */
265 	int                last_chunks_delete_boundary; /* number of chunks after last deletion */
266 	int                last_chunks_delete_count;    /* number of deletion over the last boundary */
267 #if ZEND_MM_CUSTOM
268 	union {
269 		struct {
270 			void      *(*_malloc)(size_t);
271 			void       (*_free)(void*);
272 			void      *(*_realloc)(void*, size_t);
273 		} std;
274 		struct {
275 			void      *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
276 			void       (*_free)(void*  ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
277 			void      *(*_realloc)(void*, size_t  ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
278 		} debug;
279 	} custom_heap;
280 	HashTable *tracked_allocs;
281 #endif
282 };
283 
284 struct _zend_mm_chunk {
285 	zend_mm_heap      *heap;
286 	zend_mm_chunk     *next;
287 	zend_mm_chunk     *prev;
288 	uint32_t           free_pages;				/* number of free pages */
289 	uint32_t           free_tail;               /* number of free pages at the end of chunk */
290 	uint32_t           num;
291 	char               reserve[64 - (sizeof(void*) * 3 + sizeof(uint32_t) * 3)];
292 	zend_mm_heap       heap_slot;               /* used only in main chunk */
293 	zend_mm_page_map   free_map;                /* 512 bits or 64 bytes */
294 	zend_mm_page_info  map[ZEND_MM_PAGES];      /* 2 KB = 512 * 4 */
295 };
296 
297 struct _zend_mm_page {
298 	char               bytes[ZEND_MM_PAGE_SIZE];
299 };
300 
301 /*
302  * bin - is one or few continuous pages (up to 8) used for allocation of
303  * a particular "small size".
304  */
305 struct _zend_mm_bin {
306 	char               bytes[ZEND_MM_PAGE_SIZE * 8];
307 };
308 
309 struct _zend_mm_free_slot {
310 	zend_mm_free_slot *next_free_slot;
311 };
312 
313 struct _zend_mm_huge_list {
314 	void              *ptr;
315 	size_t             size;
316 	zend_mm_huge_list *next;
317 #if ZEND_DEBUG
318 	zend_mm_debug_info dbg;
319 #endif
320 };
321 
322 #define ZEND_MM_PAGE_ADDR(chunk, page_num) \
323 	((void*)(((zend_mm_page*)(chunk)) + (page_num)))
324 
325 #define _BIN_DATA_SIZE(num, size, elements, pages, x, y) size,
326 static const uint32_t bin_data_size[] = {
327 	ZEND_MM_BINS_INFO(_BIN_DATA_SIZE, x, y)
328 };
329 
330 #define _BIN_DATA_ELEMENTS(num, size, elements, pages, x, y) elements,
331 static const uint32_t bin_elements[] = {
332 	ZEND_MM_BINS_INFO(_BIN_DATA_ELEMENTS, x, y)
333 };
334 
335 #define _BIN_DATA_PAGES(num, size, elements, pages, x, y) pages,
336 static const uint32_t bin_pages[] = {
337 	ZEND_MM_BINS_INFO(_BIN_DATA_PAGES, x, y)
338 };
339 
340 #if ZEND_DEBUG
zend_debug_alloc_output(char * format,...)341 ZEND_COLD void zend_debug_alloc_output(char *format, ...)
342 {
343 	char output_buf[256];
344 	va_list args;
345 
346 	va_start(args, format);
347 	vsprintf(output_buf, format, args);
348 	va_end(args);
349 
350 #ifdef ZEND_WIN32
351 	OutputDebugString(output_buf);
352 #else
353 	fprintf(stderr, "%s", output_buf);
354 #endif
355 }
356 #endif
357 
zend_mm_panic(const char * message)358 static ZEND_COLD ZEND_NORETURN void zend_mm_panic(const char *message)
359 {
360 	fprintf(stderr, "%s\n", message);
361 /* See http://support.microsoft.com/kb/190351 */
362 #ifdef ZEND_WIN32
363 	fflush(stderr);
364 #endif
365 #if ZEND_DEBUG && defined(HAVE_KILL) && defined(HAVE_GETPID)
366 	kill(getpid(), SIGSEGV);
367 #endif
368 	abort();
369 }
370 
zend_mm_safe_error(zend_mm_heap * heap,const char * format,size_t limit,const char * filename,uint32_t lineno,size_t size)371 static ZEND_COLD ZEND_NORETURN void zend_mm_safe_error(zend_mm_heap *heap,
372 	const char *format,
373 	size_t limit,
374 #if ZEND_DEBUG
375 	const char *filename,
376 	uint32_t lineno,
377 #endif
378 	size_t size)
379 {
380 
381 	heap->overflow = 1;
382 	zend_try {
383 		zend_error_noreturn(E_ERROR,
384 			format,
385 			limit,
386 #if ZEND_DEBUG
387 			filename,
388 			lineno,
389 #endif
390 			size);
391 	} zend_catch {
392 	}  zend_end_try();
393 	heap->overflow = 0;
394 	zend_bailout();
395 	exit(1);
396 }
397 
398 #ifdef _WIN32
399 void
stderr_last_error(char * msg)400 stderr_last_error(char *msg)
401 {
402 	DWORD err = GetLastError();
403 	char *buf = php_win32_error_to_msg(err);
404 
405 	if (!buf[0]) {
406 		fprintf(stderr, "\n%s: [0x%08lx]\n", msg, err);
407 	}
408 	else {
409 		fprintf(stderr, "\n%s: [0x%08lx] %s\n", msg, err, buf);
410 	}
411 
412 	php_win32_error_msg_free(buf);
413 }
414 #endif
415 
416 /*****************/
417 /* OS Allocation */
418 /*****************/
419 
zend_mm_munmap(void * addr,size_t size)420 static void zend_mm_munmap(void *addr, size_t size)
421 {
422 #ifdef _WIN32
423 	if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
424 		/** ERROR_INVALID_ADDRESS is expected when addr is not range start address */
425 		if (GetLastError() != ERROR_INVALID_ADDRESS) {
426 #if ZEND_MM_ERROR
427 			stderr_last_error("VirtualFree() failed");
428 #endif
429 			return;
430 		}
431 		SetLastError(0);
432 
433 		MEMORY_BASIC_INFORMATION mbi;
434 		if (VirtualQuery(addr, &mbi, sizeof(mbi)) == 0) {
435 #if ZEND_MM_ERROR
436 			stderr_last_error("VirtualQuery() failed");
437 #endif
438 			return;
439 		}
440 		addr = mbi.AllocationBase;
441 
442 		if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
443 #if ZEND_MM_ERROR
444 			stderr_last_error("VirtualFree() failed");
445 #endif
446 		}
447 	}
448 #else
449 	if (munmap(addr, size) != 0) {
450 #if ZEND_MM_ERROR
451 		fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno));
452 #endif
453 	}
454 #endif
455 }
456 
457 #ifndef HAVE_MREMAP
zend_mm_mmap_fixed(void * addr,size_t size)458 static void *zend_mm_mmap_fixed(void *addr, size_t size)
459 {
460 #ifdef _WIN32
461 	void *ptr = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
462 
463 	if (ptr == NULL) {
464 		/** ERROR_INVALID_ADDRESS is expected when fixed addr range is not free */
465 		if (GetLastError() != ERROR_INVALID_ADDRESS) {
466 #if ZEND_MM_ERROR
467 			stderr_last_error("VirtualAlloc() fixed failed");
468 #endif
469 		}
470 		SetLastError(0);
471 		return NULL;
472 	}
473 	ZEND_ASSERT(ptr == addr);
474 	return ptr;
475 #else
476 	int flags = MAP_PRIVATE | MAP_ANON;
477 #if defined(MAP_EXCL)
478 	flags |= MAP_FIXED | MAP_EXCL;
479 #elif defined(MAP_TRYFIXED)
480 	flags |= MAP_TRYFIXED;
481 #endif
482 	/* MAP_FIXED leads to discarding of the old mapping, so it can't be used. */
483 	void *ptr = mmap(addr, size, PROT_READ | PROT_WRITE, flags /*| MAP_POPULATE | MAP_HUGETLB*/, ZEND_MM_FD, 0);
484 
485 	if (ptr == MAP_FAILED) {
486 #if ZEND_MM_ERROR && !defined(MAP_EXCL) && !defined(MAP_TRYFIXED)
487 		fprintf(stderr, "\nmmap() fixed failed: [%d] %s\n", errno, strerror(errno));
488 #endif
489 		return NULL;
490 	} else if (ptr != addr) {
491 		zend_mm_munmap(ptr, size);
492 		return NULL;
493 	}
494 	return ptr;
495 #endif
496 }
497 #endif
498 
zend_mm_mmap(size_t size)499 static void *zend_mm_mmap(size_t size)
500 {
501 #ifdef _WIN32
502 	void *ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
503 
504 	if (ptr == NULL) {
505 #if ZEND_MM_ERROR
506 		stderr_last_error("VirtualAlloc() failed");
507 #endif
508 		return NULL;
509 	}
510 	return ptr;
511 #else
512 	void *ptr;
513 
514 #if defined(MAP_HUGETLB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB)
515 	if (zend_mm_use_huge_pages && size == ZEND_MM_CHUNK_SIZE) {
516 		int fd = -1;
517 		int mflags = MAP_PRIVATE | MAP_ANON;
518 #if defined(MAP_HUGETLB)
519 		mflags |= MAP_HUGETLB;
520 #else
521 		fd = VM_FLAGS_SUPERPAGE_SIZE_2MB;
522 #endif
523 		ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, mflags, fd, 0);
524 		if (ptr != MAP_FAILED) {
525 			zend_mmap_set_name(ptr, size, "zend_alloc");
526 			return ptr;
527 		}
528 	}
529 #endif
530 
531 	ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, ZEND_MM_FD, 0);
532 
533 	if (ptr == MAP_FAILED) {
534 #if ZEND_MM_ERROR
535 		fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno));
536 #endif
537 		return NULL;
538 	}
539 	zend_mmap_set_name(ptr, size, "zend_alloc");
540 	return ptr;
541 #endif
542 }
543 
544 /***********/
545 /* Bitmask */
546 /***********/
547 
548 /* number of trailing set (1) bits */
zend_mm_bitset_nts(zend_mm_bitset bitset)549 static zend_always_inline int zend_mm_bitset_nts(zend_mm_bitset bitset)
550 {
551 #if (defined(__GNUC__) || __has_builtin(__builtin_ctzl)) && SIZEOF_ZEND_LONG == SIZEOF_LONG && defined(PHP_HAVE_BUILTIN_CTZL)
552 	return __builtin_ctzl(~bitset);
553 #elif (defined(__GNUC__) || __has_builtin(__builtin_ctzll)) && defined(PHP_HAVE_BUILTIN_CTZLL)
554 	return __builtin_ctzll(~bitset);
555 #elif defined(_WIN32)
556 	unsigned long index;
557 
558 #if defined(_WIN64)
559 	if (!BitScanForward64(&index, ~bitset)) {
560 #else
561 	if (!BitScanForward(&index, ~bitset)) {
562 #endif
563 		/* undefined behavior */
564 		return 32;
565 	}
566 
567 	return (int)index;
568 #else
569 	int n;
570 
571 	if (bitset == (zend_mm_bitset)-1) return ZEND_MM_BITSET_LEN;
572 
573 	n = 0;
574 #if SIZEOF_ZEND_LONG == 8
575 	if (sizeof(zend_mm_bitset) == 8) {
576 		if ((bitset & 0xffffffff) == 0xffffffff) {n += 32; bitset = bitset >> Z_UL(32);}
577 	}
578 #endif
579 	if ((bitset & 0x0000ffff) == 0x0000ffff) {n += 16; bitset = bitset >> 16;}
580 	if ((bitset & 0x000000ff) == 0x000000ff) {n +=  8; bitset = bitset >>  8;}
581 	if ((bitset & 0x0000000f) == 0x0000000f) {n +=  4; bitset = bitset >>  4;}
582 	if ((bitset & 0x00000003) == 0x00000003) {n +=  2; bitset = bitset >>  2;}
583 	return n + (bitset & 1);
584 #endif
585 }
586 
587 static zend_always_inline int zend_mm_bitset_is_set(zend_mm_bitset *bitset, int bit)
588 {
589 	return ZEND_BIT_TEST(bitset, bit);
590 }
591 
592 static zend_always_inline void zend_mm_bitset_set_bit(zend_mm_bitset *bitset, int bit)
593 {
594 	bitset[bit / ZEND_MM_BITSET_LEN] |= (Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
595 }
596 
597 static zend_always_inline void zend_mm_bitset_reset_bit(zend_mm_bitset *bitset, int bit)
598 {
599 	bitset[bit / ZEND_MM_BITSET_LEN] &= ~(Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
600 }
601 
602 static zend_always_inline void zend_mm_bitset_set_range(zend_mm_bitset *bitset, int start, int len)
603 {
604 	if (len == 1) {
605 		zend_mm_bitset_set_bit(bitset, start);
606 	} else {
607 		int pos = start / ZEND_MM_BITSET_LEN;
608 		int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
609 		int bit = start & (ZEND_MM_BITSET_LEN - 1);
610 		zend_mm_bitset tmp;
611 
612 		if (pos != end) {
613 			/* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
614 			tmp = (zend_mm_bitset)-1 << bit;
615 			bitset[pos++] |= tmp;
616 			while (pos != end) {
617 				/* set all bits */
618 				bitset[pos++] = (zend_mm_bitset)-1;
619 			}
620 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
621 			/* set bits from "0" to "end" */
622 			tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
623 			bitset[pos] |= tmp;
624 		} else {
625 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
626 			/* set bits from "bit" to "end" */
627 			tmp = (zend_mm_bitset)-1 << bit;
628 			tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
629 			bitset[pos] |= tmp;
630 		}
631 	}
632 }
633 
634 static zend_always_inline void zend_mm_bitset_reset_range(zend_mm_bitset *bitset, int start, int len)
635 {
636 	if (len == 1) {
637 		zend_mm_bitset_reset_bit(bitset, start);
638 	} else {
639 		int pos = start / ZEND_MM_BITSET_LEN;
640 		int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
641 		int bit = start & (ZEND_MM_BITSET_LEN - 1);
642 		zend_mm_bitset tmp;
643 
644 		if (pos != end) {
645 			/* reset bits from "bit" to ZEND_MM_BITSET_LEN-1 */
646 			tmp = ~((Z_UL(1) << bit) - 1);
647 			bitset[pos++] &= ~tmp;
648 			while (pos != end) {
649 				/* set all bits */
650 				bitset[pos++] = 0;
651 			}
652 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
653 			/* reset bits from "0" to "end" */
654 			tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
655 			bitset[pos] &= ~tmp;
656 		} else {
657 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
658 			/* reset bits from "bit" to "end" */
659 			tmp = (zend_mm_bitset)-1 << bit;
660 			tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
661 			bitset[pos] &= ~tmp;
662 		}
663 	}
664 }
665 
666 static zend_always_inline int zend_mm_bitset_is_free_range(zend_mm_bitset *bitset, int start, int len)
667 {
668 	if (len == 1) {
669 		return !zend_mm_bitset_is_set(bitset, start);
670 	} else {
671 		int pos = start / ZEND_MM_BITSET_LEN;
672 		int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
673 		int bit = start & (ZEND_MM_BITSET_LEN - 1);
674 		zend_mm_bitset tmp;
675 
676 		if (pos != end) {
677 			/* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
678 			tmp = (zend_mm_bitset)-1 << bit;
679 			if ((bitset[pos++] & tmp) != 0) {
680 				return 0;
681 			}
682 			while (pos != end) {
683 				/* set all bits */
684 				if (bitset[pos++] != 0) {
685 					return 0;
686 				}
687 			}
688 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
689 			/* set bits from "0" to "end" */
690 			tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
691 			return (bitset[pos] & tmp) == 0;
692 		} else {
693 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
694 			/* set bits from "bit" to "end" */
695 			tmp = (zend_mm_bitset)-1 << bit;
696 			tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
697 			return (bitset[pos] & tmp) == 0;
698 		}
699 	}
700 }
701 
702 /**********/
703 /* Chunks */
704 /**********/
705 
706 static zend_always_inline void zend_mm_hugepage(void* ptr, size_t size)
707 {
708 #if defined(MADV_HUGEPAGE)
709 	(void)madvise(ptr, size, MADV_HUGEPAGE);
710 #elif defined(HAVE_MEMCNTL)
711 	struct memcntl_mha m = {.mha_cmd = MHA_MAPSIZE_VA, .mha_pagesize = ZEND_MM_CHUNK_SIZE, .mha_flags = 0};
712 	(void)memcntl(ptr, size, MC_HAT_ADVISE, (char *)&m, 0, 0);
713 #elif !defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && !defined(MAP_ALIGNED_SUPER)
714 	zend_error_noreturn(E_WARNING, "huge_pages: thp unsupported on this platform");
715 #endif
716 }
717 
718 static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment)
719 {
720 	void *ptr = zend_mm_mmap(size);
721 
722 	if (ptr == NULL) {
723 		return NULL;
724 	} else if (ZEND_MM_ALIGNED_OFFSET(ptr, alignment) == 0) {
725 		if (zend_mm_use_huge_pages) {
726 			zend_mm_hugepage(ptr, size);
727 		}
728 		return ptr;
729 	} else {
730 		size_t offset;
731 
732 		/* chunk has to be aligned */
733 		zend_mm_munmap(ptr, size);
734 		ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
735 #ifdef _WIN32
736 		offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
737 		if (offset != 0) {
738 			offset = alignment - offset;
739 		}
740 		zend_mm_munmap(ptr, size + alignment - REAL_PAGE_SIZE);
741 		ptr = zend_mm_mmap_fixed((void*)((char*)ptr + offset), size);
742 		if (ptr == NULL) { // fix GH-9650, fixed addr range is not free
743 			ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
744 			if (ptr == NULL) {
745 				return NULL;
746 			}
747 			offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
748 			if (offset != 0) {
749 				ptr = (void*)((char*)ptr + alignment - offset);
750 			}
751 		}
752 		return ptr;
753 #else
754 		offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
755 		if (offset != 0) {
756 			offset = alignment - offset;
757 			zend_mm_munmap(ptr, offset);
758 			ptr = (char*)ptr + offset;
759 			alignment -= offset;
760 		}
761 		if (alignment > REAL_PAGE_SIZE) {
762 			zend_mm_munmap((char*)ptr + size, alignment - REAL_PAGE_SIZE);
763 		}
764 		if (zend_mm_use_huge_pages) {
765 			zend_mm_hugepage(ptr, size);
766 		}
767 #endif
768 		return ptr;
769 	}
770 }
771 
772 static void *zend_mm_chunk_alloc(zend_mm_heap *heap, size_t size, size_t alignment)
773 {
774 #if ZEND_MM_STORAGE
775 	if (UNEXPECTED(heap->storage)) {
776 		void *ptr = heap->storage->handlers.chunk_alloc(heap->storage, size, alignment);
777 		ZEND_ASSERT(((zend_uintptr_t)((char*)ptr + (alignment-1)) & (alignment-1)) == (zend_uintptr_t)ptr);
778 		return ptr;
779 	}
780 #endif
781 	return zend_mm_chunk_alloc_int(size, alignment);
782 }
783 
784 static void zend_mm_chunk_free(zend_mm_heap *heap, void *addr, size_t size)
785 {
786 #if ZEND_MM_STORAGE
787 	if (UNEXPECTED(heap->storage)) {
788 		heap->storage->handlers.chunk_free(heap->storage, addr, size);
789 		return;
790 	}
791 #endif
792 	zend_mm_munmap(addr, size);
793 }
794 
795 static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
796 {
797 #if ZEND_MM_STORAGE
798 	if (UNEXPECTED(heap->storage)) {
799 		if (heap->storage->handlers.chunk_truncate) {
800 			return heap->storage->handlers.chunk_truncate(heap->storage, addr, old_size, new_size);
801 		} else {
802 			return 0;
803 		}
804 	}
805 #endif
806 #ifndef _WIN32
807 	zend_mm_munmap((char*)addr + new_size, old_size - new_size);
808 	return 1;
809 #else
810 	return 0;
811 #endif
812 }
813 
814 static int zend_mm_chunk_extend(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
815 {
816 #if ZEND_MM_STORAGE
817 	if (UNEXPECTED(heap->storage)) {
818 		if (heap->storage->handlers.chunk_extend) {
819 			return heap->storage->handlers.chunk_extend(heap->storage, addr, old_size, new_size);
820 		} else {
821 			return 0;
822 		}
823 	}
824 #endif
825 #ifdef HAVE_MREMAP
826 	/* We don't use MREMAP_MAYMOVE due to alignment requirements. */
827 	void *ptr = mremap(addr, old_size, new_size, 0);
828 	if (ptr == MAP_FAILED) {
829 		return 0;
830 	}
831 	/* Sanity check: The mapping shouldn't have moved. */
832 	ZEND_ASSERT(ptr == addr);
833 	return 1;
834 #elif !defined(_WIN32)
835 	return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL);
836 #else
837 	return 0;
838 #endif
839 }
840 
841 static zend_always_inline void zend_mm_chunk_init(zend_mm_heap *heap, zend_mm_chunk *chunk)
842 {
843 	chunk->heap = heap;
844 	chunk->next = heap->main_chunk;
845 	chunk->prev = heap->main_chunk->prev;
846 	chunk->prev->next = chunk;
847 	chunk->next->prev = chunk;
848 	/* mark first pages as allocated */
849 	chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
850 	chunk->free_tail = ZEND_MM_FIRST_PAGE;
851 	/* the younger chunks have bigger number */
852 	chunk->num = chunk->prev->num + 1;
853 	/* mark first pages as allocated */
854 	chunk->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
855 	chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
856 }
857 
858 /***********************/
859 /* Huge Runs (forward) */
860 /***********************/
861 
862 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
863 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
864 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
865 
866 #if ZEND_DEBUG
867 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
868 #else
869 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
870 #endif
871 
872 /**************/
873 /* Large Runs */
874 /**************/
875 
876 #if ZEND_DEBUG
877 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
878 #else
879 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
880 #endif
881 {
882 	zend_mm_chunk *chunk = heap->main_chunk;
883 	uint32_t page_num, len;
884 	int steps = 0;
885 
886 	while (1) {
887 		if (UNEXPECTED(chunk->free_pages < pages_count)) {
888 			goto not_found;
889 #if 0
890 		} else if (UNEXPECTED(chunk->free_pages + chunk->free_tail == ZEND_MM_PAGES)) {
891 			if (UNEXPECTED(ZEND_MM_PAGES - chunk->free_tail < pages_count)) {
892 				goto not_found;
893 			} else {
894 				page_num = chunk->free_tail;
895 				goto found;
896 			}
897 		} else if (0) {
898 			/* First-Fit Search */
899 			int free_tail = chunk->free_tail;
900 			zend_mm_bitset *bitset = chunk->free_map;
901 			zend_mm_bitset tmp = *(bitset++);
902 			int i = 0;
903 
904 			while (1) {
905 				/* skip allocated blocks */
906 				while (tmp == (zend_mm_bitset)-1) {
907 					i += ZEND_MM_BITSET_LEN;
908 					if (i == ZEND_MM_PAGES) {
909 						goto not_found;
910 					}
911 					tmp = *(bitset++);
912 				}
913 				/* find first 0 bit */
914 				page_num = i + zend_mm_bitset_nts(tmp);
915 				/* reset bits from 0 to "bit" */
916 				tmp &= tmp + 1;
917 				/* skip free blocks */
918 				while (tmp == 0) {
919 					i += ZEND_MM_BITSET_LEN;
920 					len = i - page_num;
921 					if (len >= pages_count) {
922 						goto found;
923 					} else if (i >= free_tail) {
924 						goto not_found;
925 					}
926 					tmp = *(bitset++);
927 				}
928 				/* find first 1 bit */
929 				len = (i + zend_ulong_ntz(tmp)) - page_num;
930 				if (len >= pages_count) {
931 					goto found;
932 				}
933 				/* set bits from 0 to "bit" */
934 				tmp |= tmp - 1;
935 			}
936 #endif
937 		} else {
938 			/* Best-Fit Search */
939 			int best = -1;
940 			uint32_t best_len = ZEND_MM_PAGES;
941 			uint32_t free_tail = chunk->free_tail;
942 			zend_mm_bitset *bitset = chunk->free_map;
943 			zend_mm_bitset tmp = *(bitset++);
944 			uint32_t i = 0;
945 
946 			while (1) {
947 				/* skip allocated blocks */
948 				while (tmp == (zend_mm_bitset)-1) {
949 					i += ZEND_MM_BITSET_LEN;
950 					if (i == ZEND_MM_PAGES) {
951 						if (best > 0) {
952 							page_num = best;
953 							goto found;
954 						} else {
955 							goto not_found;
956 						}
957 					}
958 					tmp = *(bitset++);
959 				}
960 				/* find first 0 bit */
961 				page_num = i + zend_mm_bitset_nts(tmp);
962 				/* reset bits from 0 to "bit" */
963 				tmp &= tmp + 1;
964 				/* skip free blocks */
965 				while (tmp == 0) {
966 					i += ZEND_MM_BITSET_LEN;
967 					if (i >= free_tail || i == ZEND_MM_PAGES) {
968 						len = ZEND_MM_PAGES - page_num;
969 						if (len >= pages_count && len < best_len) {
970 							chunk->free_tail = page_num + pages_count;
971 							goto found;
972 						} else {
973 							/* set accurate value */
974 							chunk->free_tail = page_num;
975 							if (best > 0) {
976 								page_num = best;
977 								goto found;
978 							} else {
979 								goto not_found;
980 							}
981 						}
982 					}
983 					tmp = *(bitset++);
984 				}
985 				/* find first 1 bit */
986 				len = i + zend_ulong_ntz(tmp) - page_num;
987 				if (len >= pages_count) {
988 					if (len == pages_count) {
989 						goto found;
990 					} else if (len < best_len) {
991 						best_len = len;
992 						best = page_num;
993 					}
994 				}
995 				/* set bits from 0 to "bit" */
996 				tmp |= tmp - 1;
997 			}
998 		}
999 
1000 not_found:
1001 		if (chunk->next == heap->main_chunk) {
1002 get_chunk:
1003 			if (heap->cached_chunks) {
1004 				heap->cached_chunks_count--;
1005 				chunk = heap->cached_chunks;
1006 				heap->cached_chunks = chunk->next;
1007 			} else {
1008 #if ZEND_MM_LIMIT
1009 				if (UNEXPECTED(ZEND_MM_CHUNK_SIZE > heap->limit - heap->real_size)) {
1010 					if (zend_mm_gc(heap)) {
1011 						goto get_chunk;
1012 					} else if (heap->overflow == 0) {
1013 #if ZEND_DEBUG
1014 						zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1015 #else
1016 						zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, ZEND_MM_PAGE_SIZE * pages_count);
1017 #endif
1018 						return NULL;
1019 					}
1020 				}
1021 #endif
1022 				chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1023 				if (UNEXPECTED(chunk == NULL)) {
1024 					/* insufficient memory */
1025 					if (zend_mm_gc(heap) &&
1026 					    (chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE)) != NULL) {
1027 						/* pass */
1028 					} else {
1029 #if !ZEND_MM_LIMIT
1030 						zend_mm_safe_error(heap, "Out of memory");
1031 #elif ZEND_DEBUG
1032 						zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1033 #else
1034 						zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, ZEND_MM_PAGE_SIZE * pages_count);
1035 #endif
1036 						return NULL;
1037 					}
1038 				}
1039 #if ZEND_MM_STAT
1040 				do {
1041 					size_t size = heap->real_size + ZEND_MM_CHUNK_SIZE;
1042 					size_t peak = MAX(heap->real_peak, size);
1043 					heap->real_size = size;
1044 					heap->real_peak = peak;
1045 				} while (0);
1046 #elif ZEND_MM_LIMIT
1047 				heap->real_size += ZEND_MM_CHUNK_SIZE;
1048 
1049 #endif
1050 			}
1051 			heap->chunks_count++;
1052 			if (heap->chunks_count > heap->peak_chunks_count) {
1053 				heap->peak_chunks_count = heap->chunks_count;
1054 			}
1055 			zend_mm_chunk_init(heap, chunk);
1056 			page_num = ZEND_MM_FIRST_PAGE;
1057 			len = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1058 			goto found;
1059 		} else {
1060 			chunk = chunk->next;
1061 			steps++;
1062 		}
1063 	}
1064 
1065 found:
1066 	if (steps > 2 && pages_count < 8) {
1067 		/* move chunk into the head of the linked-list */
1068 		chunk->prev->next = chunk->next;
1069 		chunk->next->prev = chunk->prev;
1070 		chunk->next = heap->main_chunk->next;
1071 		chunk->prev = heap->main_chunk;
1072 		chunk->prev->next = chunk;
1073 		chunk->next->prev = chunk;
1074 	}
1075 	/* mark run as allocated */
1076 	chunk->free_pages -= pages_count;
1077 	zend_mm_bitset_set_range(chunk->free_map, page_num, pages_count);
1078 	chunk->map[page_num] = ZEND_MM_LRUN(pages_count);
1079 	if (page_num == chunk->free_tail) {
1080 		chunk->free_tail = page_num + pages_count;
1081 	}
1082 	return ZEND_MM_PAGE_ADDR(chunk, page_num);
1083 }
1084 
1085 static zend_always_inline void *zend_mm_alloc_large_ex(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1086 {
1087 	int pages_count = (int)ZEND_MM_SIZE_TO_NUM(size, ZEND_MM_PAGE_SIZE);
1088 #if ZEND_DEBUG
1089 	void *ptr = zend_mm_alloc_pages(heap, pages_count, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1090 #else
1091 	void *ptr = zend_mm_alloc_pages(heap, pages_count ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1092 #endif
1093 #if ZEND_MM_STAT
1094 	do {
1095 		size_t size = heap->size + pages_count * ZEND_MM_PAGE_SIZE;
1096 		size_t peak = MAX(heap->peak, size);
1097 		heap->size = size;
1098 		heap->peak = peak;
1099 	} while (0);
1100 #endif
1101 	return ptr;
1102 }
1103 
1104 #if ZEND_DEBUG
1105 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1106 {
1107 	return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1108 }
1109 #else
1110 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1111 {
1112 	return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1113 }
1114 #endif
1115 
1116 static zend_always_inline void zend_mm_delete_chunk(zend_mm_heap *heap, zend_mm_chunk *chunk)
1117 {
1118 	chunk->next->prev = chunk->prev;
1119 	chunk->prev->next = chunk->next;
1120 	heap->chunks_count--;
1121 	if (heap->chunks_count + heap->cached_chunks_count < heap->avg_chunks_count + 0.1
1122 	 || (heap->chunks_count == heap->last_chunks_delete_boundary
1123 	  && heap->last_chunks_delete_count >= 4)) {
1124 		/* delay deletion */
1125 		heap->cached_chunks_count++;
1126 		chunk->next = heap->cached_chunks;
1127 		heap->cached_chunks = chunk;
1128 	} else {
1129 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1130 		heap->real_size -= ZEND_MM_CHUNK_SIZE;
1131 #endif
1132 		if (!heap->cached_chunks) {
1133 			if (heap->chunks_count != heap->last_chunks_delete_boundary) {
1134 				heap->last_chunks_delete_boundary = heap->chunks_count;
1135 				heap->last_chunks_delete_count = 0;
1136 			} else {
1137 				heap->last_chunks_delete_count++;
1138 			}
1139 		}
1140 		if (!heap->cached_chunks || chunk->num > heap->cached_chunks->num) {
1141 			zend_mm_chunk_free(heap, chunk, ZEND_MM_CHUNK_SIZE);
1142 		} else {
1143 //TODO: select the best chunk to delete???
1144 			chunk->next = heap->cached_chunks->next;
1145 			zend_mm_chunk_free(heap, heap->cached_chunks, ZEND_MM_CHUNK_SIZE);
1146 			heap->cached_chunks = chunk;
1147 		}
1148 	}
1149 }
1150 
1151 static zend_always_inline void zend_mm_free_pages_ex(zend_mm_heap *heap, zend_mm_chunk *chunk, uint32_t page_num, uint32_t pages_count, int free_chunk)
1152 {
1153 	chunk->free_pages += pages_count;
1154 	zend_mm_bitset_reset_range(chunk->free_map, page_num, pages_count);
1155 	chunk->map[page_num] = 0;
1156 	if (chunk->free_tail == page_num + pages_count) {
1157 		/* this setting may be not accurate */
1158 		chunk->free_tail = page_num;
1159 	}
1160 	if (free_chunk && chunk != heap->main_chunk && chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
1161 		zend_mm_delete_chunk(heap, chunk);
1162 	}
1163 }
1164 
1165 static zend_never_inline void zend_mm_free_pages(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1166 {
1167 	zend_mm_free_pages_ex(heap, chunk, page_num, pages_count, 1);
1168 }
1169 
1170 static zend_always_inline void zend_mm_free_large(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1171 {
1172 #if ZEND_MM_STAT
1173 	heap->size -= pages_count * ZEND_MM_PAGE_SIZE;
1174 #endif
1175 	zend_mm_free_pages(heap, chunk, page_num, pages_count);
1176 }
1177 
1178 /**************/
1179 /* Small Runs */
1180 /**************/
1181 
1182 /* higher set bit number (0->N/A, 1->1, 2->2, 4->3, 8->4, 127->7, 128->8 etc) */
1183 static zend_always_inline int zend_mm_small_size_to_bit(int size)
1184 {
1185 #if (defined(__GNUC__) || __has_builtin(__builtin_clz))  && defined(PHP_HAVE_BUILTIN_CLZ)
1186 	return (__builtin_clz(size) ^ 0x1f) + 1;
1187 #elif defined(_WIN32)
1188 	unsigned long index;
1189 
1190 	if (!BitScanReverse(&index, (unsigned long)size)) {
1191 		/* undefined behavior */
1192 		return 64;
1193 	}
1194 
1195 	return (((31 - (int)index) ^ 0x1f) + 1);
1196 #else
1197 	int n = 16;
1198 	if (size <= 0x00ff) {n -= 8; size = size << 8;}
1199 	if (size <= 0x0fff) {n -= 4; size = size << 4;}
1200 	if (size <= 0x3fff) {n -= 2; size = size << 2;}
1201 	if (size <= 0x7fff) {n -= 1;}
1202 	return n;
1203 #endif
1204 }
1205 
1206 #ifndef MAX
1207 # define MAX(a, b) (((a) > (b)) ? (a) : (b))
1208 #endif
1209 
1210 #ifndef MIN
1211 # define MIN(a, b) (((a) < (b)) ? (a) : (b))
1212 #endif
1213 
1214 static zend_always_inline int zend_mm_small_size_to_bin(size_t size)
1215 {
1216 #if 0
1217 	int n;
1218                             /*0,  1,  2,  3,  4,  5,  6,  7,  8,  9  10, 11, 12*/
1219 	static const int f1[] = { 3,  3,  3,  3,  3,  3,  3,  4,  5,  6,  7,  8,  9};
1220 	static const int f2[] = { 0,  0,  0,  0,  0,  0,  0,  4,  8, 12, 16, 20, 24};
1221 
1222 	if (UNEXPECTED(size <= 2)) return 0;
1223 	n = zend_mm_small_size_to_bit(size - 1);
1224 	return ((size-1) >> f1[n]) + f2[n];
1225 #else
1226 	unsigned int t1, t2;
1227 
1228 	if (size <= 64) {
1229 		/* we need to support size == 0 ... */
1230 		return (size - !!size) >> 3;
1231 	} else {
1232 		t1 = size - 1;
1233 		t2 = zend_mm_small_size_to_bit(t1) - 3;
1234 		t1 = t1 >> t2;
1235 		t2 = t2 - 3;
1236 		t2 = t2 << 2;
1237 		return (int)(t1 + t2);
1238 	}
1239 #endif
1240 }
1241 
1242 #define ZEND_MM_SMALL_SIZE_TO_BIN(size)  zend_mm_small_size_to_bin(size)
1243 
1244 static zend_never_inline void *zend_mm_alloc_small_slow(zend_mm_heap *heap, uint32_t bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1245 {
1246 	zend_mm_chunk *chunk;
1247 	int page_num;
1248 	zend_mm_bin *bin;
1249 	zend_mm_free_slot *p, *end;
1250 
1251 #if ZEND_DEBUG
1252 	bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num], bin_data_size[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1253 #else
1254 	bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1255 #endif
1256 	if (UNEXPECTED(bin == NULL)) {
1257 		/* insufficient memory */
1258 		return NULL;
1259 	}
1260 
1261 	chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(bin, ZEND_MM_CHUNK_SIZE);
1262 	page_num = ZEND_MM_ALIGNED_OFFSET(bin, ZEND_MM_CHUNK_SIZE) / ZEND_MM_PAGE_SIZE;
1263 	chunk->map[page_num] = ZEND_MM_SRUN(bin_num);
1264 	if (bin_pages[bin_num] > 1) {
1265 		uint32_t i = 1;
1266 
1267 		do {
1268 			chunk->map[page_num+i] = ZEND_MM_NRUN(bin_num, i);
1269 			i++;
1270 		} while (i < bin_pages[bin_num]);
1271 	}
1272 
1273 	/* create a linked list of elements from 1 to last */
1274 	end = (zend_mm_free_slot*)((char*)bin + (bin_data_size[bin_num] * (bin_elements[bin_num] - 1)));
1275 	heap->free_slot[bin_num] = p = (zend_mm_free_slot*)((char*)bin + bin_data_size[bin_num]);
1276 	do {
1277 		p->next_free_slot = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1278 #if ZEND_DEBUG
1279 		do {
1280 			zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1281 			dbg->size = 0;
1282 		} while (0);
1283 #endif
1284 		p = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1285 	} while (p != end);
1286 
1287 	/* terminate list using NULL */
1288 	p->next_free_slot = NULL;
1289 #if ZEND_DEBUG
1290 		do {
1291 			zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1292 			dbg->size = 0;
1293 		} while (0);
1294 #endif
1295 
1296 	/* return first element */
1297 	return bin;
1298 }
1299 
1300 static zend_always_inline void *zend_mm_alloc_small(zend_mm_heap *heap, int bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1301 {
1302 #if ZEND_MM_STAT
1303 	do {
1304 		size_t size = heap->size + bin_data_size[bin_num];
1305 		size_t peak = MAX(heap->peak, size);
1306 		heap->size = size;
1307 		heap->peak = peak;
1308 	} while (0);
1309 #endif
1310 
1311 	if (EXPECTED(heap->free_slot[bin_num] != NULL)) {
1312 		zend_mm_free_slot *p = heap->free_slot[bin_num];
1313 		heap->free_slot[bin_num] = p->next_free_slot;
1314 		return p;
1315 	} else {
1316 		return zend_mm_alloc_small_slow(heap, bin_num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1317 	}
1318 }
1319 
1320 static zend_always_inline void zend_mm_free_small(zend_mm_heap *heap, void *ptr, int bin_num)
1321 {
1322 	zend_mm_free_slot *p;
1323 
1324 #if ZEND_MM_STAT
1325 	heap->size -= bin_data_size[bin_num];
1326 #endif
1327 
1328 #if ZEND_DEBUG
1329 	do {
1330 		zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1331 		dbg->size = 0;
1332 	} while (0);
1333 #endif
1334 
1335 	p = (zend_mm_free_slot*)ptr;
1336 	p->next_free_slot = heap->free_slot[bin_num];
1337 	heap->free_slot[bin_num] = p;
1338 }
1339 
1340 /********/
1341 /* Heap */
1342 /********/
1343 
1344 #if ZEND_DEBUG
1345 static zend_always_inline zend_mm_debug_info *zend_mm_get_debug_info(zend_mm_heap *heap, void *ptr)
1346 {
1347 	size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1348 	zend_mm_chunk *chunk;
1349 	int page_num;
1350 	zend_mm_page_info info;
1351 
1352 	ZEND_MM_CHECK(page_offset != 0, "zend_mm_heap corrupted");
1353 	chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1354 	page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1355 	info = chunk->map[page_num];
1356 	ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1357 	if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1358 		int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1359 		return (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1360 	} else /* if (info & ZEND_MM_IS_LRUN) */ {
1361 		int pages_count = ZEND_MM_LRUN_PAGES(info);
1362 
1363 		return (zend_mm_debug_info*)((char*)ptr + ZEND_MM_PAGE_SIZE * pages_count - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1364 	}
1365 }
1366 #endif
1367 
1368 static zend_always_inline void *zend_mm_alloc_heap(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1369 {
1370 	void *ptr;
1371 #if ZEND_DEBUG
1372 	size_t real_size = size;
1373 	zend_mm_debug_info *dbg;
1374 
1375 	/* special handling for zero-size allocation */
1376 	size = MAX(size, 1);
1377 	size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1378 	if (UNEXPECTED(size < real_size)) {
1379 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", ZEND_MM_ALIGNED_SIZE(real_size), ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1380 		return NULL;
1381 	}
1382 #endif
1383 	if (EXPECTED(size <= ZEND_MM_MAX_SMALL_SIZE)) {
1384 		ptr = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1385 #if ZEND_DEBUG
1386 		dbg = zend_mm_get_debug_info(heap, ptr);
1387 		dbg->size = real_size;
1388 		dbg->filename = __zend_filename;
1389 		dbg->orig_filename = __zend_orig_filename;
1390 		dbg->lineno = __zend_lineno;
1391 		dbg->orig_lineno = __zend_orig_lineno;
1392 #endif
1393 		return ptr;
1394 	} else if (EXPECTED(size <= ZEND_MM_MAX_LARGE_SIZE)) {
1395 		ptr = zend_mm_alloc_large(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1396 #if ZEND_DEBUG
1397 		dbg = zend_mm_get_debug_info(heap, ptr);
1398 		dbg->size = real_size;
1399 		dbg->filename = __zend_filename;
1400 		dbg->orig_filename = __zend_orig_filename;
1401 		dbg->lineno = __zend_lineno;
1402 		dbg->orig_lineno = __zend_orig_lineno;
1403 #endif
1404 		return ptr;
1405 	} else {
1406 #if ZEND_DEBUG
1407 		size = real_size;
1408 #endif
1409 		return zend_mm_alloc_huge(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1410 	}
1411 }
1412 
1413 static zend_always_inline void zend_mm_free_heap(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1414 {
1415 	size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1416 
1417 	if (UNEXPECTED(page_offset == 0)) {
1418 		if (ptr != NULL) {
1419 			zend_mm_free_huge(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1420 		}
1421 	} else {
1422 		zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1423 		int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1424 		zend_mm_page_info info = chunk->map[page_num];
1425 
1426 		ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1427 		if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1428 			zend_mm_free_small(heap, ptr, ZEND_MM_SRUN_BIN_NUM(info));
1429 		} else /* if (info & ZEND_MM_IS_LRUN) */ {
1430 			int pages_count = ZEND_MM_LRUN_PAGES(info);
1431 
1432 			ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1433 			zend_mm_free_large(heap, chunk, page_num, pages_count);
1434 		}
1435 	}
1436 }
1437 
1438 static size_t zend_mm_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1439 {
1440 	size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1441 
1442 	if (UNEXPECTED(page_offset == 0)) {
1443 		return zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1444 	} else {
1445 		zend_mm_chunk *chunk;
1446 #if 0 && ZEND_DEBUG
1447 		zend_mm_debug_info *dbg = zend_mm_get_debug_info(heap, ptr);
1448 		return dbg->size;
1449 #else
1450 		int page_num;
1451 		zend_mm_page_info info;
1452 
1453 		chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1454 		page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1455 		info = chunk->map[page_num];
1456 		ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1457 		if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1458 			return bin_data_size[ZEND_MM_SRUN_BIN_NUM(info)];
1459 		} else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1460 			return ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1461 		}
1462 #endif
1463 	}
1464 }
1465 
1466 static zend_never_inline void *zend_mm_realloc_slow(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1467 {
1468 	void *ret;
1469 
1470 #if ZEND_MM_STAT
1471 	do {
1472 		size_t orig_peak = heap->peak;
1473 #endif
1474 		ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1475 		memcpy(ret, ptr, copy_size);
1476 		zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1477 #if ZEND_MM_STAT
1478 		heap->peak = MAX(orig_peak, heap->size);
1479 	} while (0);
1480 #endif
1481 	return ret;
1482 }
1483 
1484 static zend_never_inline void *zend_mm_realloc_huge(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1485 {
1486 	size_t old_size;
1487 	size_t new_size;
1488 #if ZEND_DEBUG
1489 	size_t real_size;
1490 #endif
1491 
1492 	old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1493 #if ZEND_DEBUG
1494 	real_size = size;
1495 	size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1496 #endif
1497 	if (size > ZEND_MM_MAX_LARGE_SIZE) {
1498 #if ZEND_DEBUG
1499 		size = real_size;
1500 #endif
1501 #ifdef ZEND_WIN32
1502 		/* On Windows we don't have ability to extend huge blocks in-place.
1503 		 * We allocate them with 2MB size granularity, to avoid many
1504 		 * reallocations when they are extended by small pieces
1505 		 */
1506 		new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE));
1507 #else
1508 		new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE);
1509 #endif
1510 		if (new_size == old_size) {
1511 #if ZEND_DEBUG
1512 			zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1513 #else
1514 			zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1515 #endif
1516 			return ptr;
1517 		} else if (new_size < old_size) {
1518 			/* unmup tail */
1519 			if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) {
1520 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1521 				heap->real_size -= old_size - new_size;
1522 #endif
1523 #if ZEND_MM_STAT
1524 				heap->size -= old_size - new_size;
1525 #endif
1526 #if ZEND_DEBUG
1527 				zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1528 #else
1529 				zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1530 #endif
1531 				return ptr;
1532 			}
1533 		} else /* if (new_size > old_size) */ {
1534 #if ZEND_MM_LIMIT
1535 			if (UNEXPECTED(new_size - old_size > heap->limit - heap->real_size)) {
1536 				if (zend_mm_gc(heap) && new_size - old_size <= heap->limit - heap->real_size) {
1537 					/* pass */
1538 				} else if (heap->overflow == 0) {
1539 #if ZEND_DEBUG
1540 					zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1541 #else
1542 					zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1543 #endif
1544 					return NULL;
1545 				}
1546 			}
1547 #endif
1548 			/* try to map tail right after this block */
1549 			if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) {
1550 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1551 				heap->real_size += new_size - old_size;
1552 #endif
1553 #if ZEND_MM_STAT
1554 				heap->real_peak = MAX(heap->real_peak, heap->real_size);
1555 				heap->size += new_size - old_size;
1556 				heap->peak = MAX(heap->peak, heap->size);
1557 #endif
1558 #if ZEND_DEBUG
1559 				zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1560 #else
1561 				zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1562 #endif
1563 				return ptr;
1564 			}
1565 		}
1566 	}
1567 
1568 	return zend_mm_realloc_slow(heap, ptr, size, MIN(old_size, copy_size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1569 }
1570 
1571 static zend_always_inline void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, bool use_copy_size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1572 {
1573 	size_t page_offset;
1574 	size_t old_size;
1575 	size_t new_size;
1576 	void *ret;
1577 #if ZEND_DEBUG
1578 	zend_mm_debug_info *dbg;
1579 #endif
1580 
1581 	page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1582 	if (UNEXPECTED(page_offset == 0)) {
1583 		if (EXPECTED(ptr == NULL)) {
1584 			return _zend_mm_alloc(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1585 		} else {
1586 			return zend_mm_realloc_huge(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1587 		}
1588 	} else {
1589 		zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1590 		int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1591 		zend_mm_page_info info = chunk->map[page_num];
1592 #if ZEND_DEBUG
1593 		size_t real_size = size;
1594 
1595 		size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1596 #endif
1597 
1598 		ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1599 		if (info & ZEND_MM_IS_SRUN) {
1600 			int old_bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1601 
1602 			do {
1603 				old_size = bin_data_size[old_bin_num];
1604 
1605 				/* Check if requested size fits into current bin */
1606 				if (size <= old_size) {
1607 					/* Check if truncation is necessary */
1608 					if (old_bin_num > 0 && size < bin_data_size[old_bin_num - 1]) {
1609 						/* truncation */
1610 						ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1611 						copy_size = use_copy_size ? MIN(size, copy_size) : size;
1612 						memcpy(ret, ptr, copy_size);
1613 						zend_mm_free_small(heap, ptr, old_bin_num);
1614 					} else {
1615 						/* reallocation in-place */
1616 						ret = ptr;
1617 					}
1618 				} else if (size <= ZEND_MM_MAX_SMALL_SIZE) {
1619 					/* small extension */
1620 
1621 #if ZEND_MM_STAT
1622 					do {
1623 						size_t orig_peak = heap->peak;
1624 #endif
1625 						ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1626 						copy_size = use_copy_size ? MIN(old_size, copy_size) : old_size;
1627 						memcpy(ret, ptr, copy_size);
1628 						zend_mm_free_small(heap, ptr, old_bin_num);
1629 #if ZEND_MM_STAT
1630 						heap->peak = MAX(orig_peak, heap->size);
1631 					} while (0);
1632 #endif
1633 				} else {
1634 					/* slow reallocation */
1635 					break;
1636 				}
1637 
1638 #if ZEND_DEBUG
1639 				dbg = zend_mm_get_debug_info(heap, ret);
1640 				dbg->size = real_size;
1641 				dbg->filename = __zend_filename;
1642 				dbg->orig_filename = __zend_orig_filename;
1643 				dbg->lineno = __zend_lineno;
1644 				dbg->orig_lineno = __zend_orig_lineno;
1645 #endif
1646 				return ret;
1647 			}  while (0);
1648 
1649 		} else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1650 			ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1651 			old_size = ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1652 			if (size > ZEND_MM_MAX_SMALL_SIZE && size <= ZEND_MM_MAX_LARGE_SIZE) {
1653 				new_size = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE);
1654 				if (new_size == old_size) {
1655 #if ZEND_DEBUG
1656 					dbg = zend_mm_get_debug_info(heap, ptr);
1657 					dbg->size = real_size;
1658 					dbg->filename = __zend_filename;
1659 					dbg->orig_filename = __zend_orig_filename;
1660 					dbg->lineno = __zend_lineno;
1661 					dbg->orig_lineno = __zend_orig_lineno;
1662 #endif
1663 					return ptr;
1664 				} else if (new_size < old_size) {
1665 					/* free tail pages */
1666 					int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1667 					int rest_pages_count = (int)((old_size - new_size) / ZEND_MM_PAGE_SIZE);
1668 
1669 #if ZEND_MM_STAT
1670 					heap->size -= rest_pages_count * ZEND_MM_PAGE_SIZE;
1671 #endif
1672 					chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1673 					chunk->free_pages += rest_pages_count;
1674 					zend_mm_bitset_reset_range(chunk->free_map, page_num + new_pages_count, rest_pages_count);
1675 #if ZEND_DEBUG
1676 					dbg = zend_mm_get_debug_info(heap, ptr);
1677 					dbg->size = real_size;
1678 					dbg->filename = __zend_filename;
1679 					dbg->orig_filename = __zend_orig_filename;
1680 					dbg->lineno = __zend_lineno;
1681 					dbg->orig_lineno = __zend_orig_lineno;
1682 #endif
1683 					return ptr;
1684 				} else /* if (new_size > old_size) */ {
1685 					int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1686 					int old_pages_count = (int)(old_size / ZEND_MM_PAGE_SIZE);
1687 
1688 					/* try to allocate tail pages after this block */
1689 					if (page_num + new_pages_count <= ZEND_MM_PAGES &&
1690 					    zend_mm_bitset_is_free_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count)) {
1691 #if ZEND_MM_STAT
1692 						do {
1693 							size_t size = heap->size + (new_size - old_size);
1694 							size_t peak = MAX(heap->peak, size);
1695 							heap->size = size;
1696 							heap->peak = peak;
1697 						} while (0);
1698 #endif
1699 						chunk->free_pages -= new_pages_count - old_pages_count;
1700 						zend_mm_bitset_set_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count);
1701 						chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1702 #if ZEND_DEBUG
1703 						dbg = zend_mm_get_debug_info(heap, ptr);
1704 						dbg->size = real_size;
1705 						dbg->filename = __zend_filename;
1706 						dbg->orig_filename = __zend_orig_filename;
1707 						dbg->lineno = __zend_lineno;
1708 						dbg->orig_lineno = __zend_orig_lineno;
1709 #endif
1710 						return ptr;
1711 					}
1712 				}
1713 			}
1714 		}
1715 #if ZEND_DEBUG
1716 		size = real_size;
1717 #endif
1718 	}
1719 
1720 	copy_size = MIN(old_size, copy_size);
1721 	return zend_mm_realloc_slow(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1722 }
1723 
1724 /*********************/
1725 /* Huge Runs (again) */
1726 /*********************/
1727 
1728 #if ZEND_DEBUG
1729 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1730 #else
1731 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1732 #endif
1733 {
1734 	zend_mm_huge_list *list = (zend_mm_huge_list*)zend_mm_alloc_heap(heap, sizeof(zend_mm_huge_list) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1735 	list->ptr = ptr;
1736 	list->size = size;
1737 	list->next = heap->huge_list;
1738 #if ZEND_DEBUG
1739 	list->dbg.size = dbg_size;
1740 	list->dbg.filename = __zend_filename;
1741 	list->dbg.orig_filename = __zend_orig_filename;
1742 	list->dbg.lineno = __zend_lineno;
1743 	list->dbg.orig_lineno = __zend_orig_lineno;
1744 #endif
1745 	heap->huge_list = list;
1746 }
1747 
1748 static size_t zend_mm_del_huge_block(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1749 {
1750 	zend_mm_huge_list *prev = NULL;
1751 	zend_mm_huge_list *list = heap->huge_list;
1752 	while (list != NULL) {
1753 		if (list->ptr == ptr) {
1754 			size_t size;
1755 
1756 			if (prev) {
1757 				prev->next = list->next;
1758 			} else {
1759 				heap->huge_list = list->next;
1760 			}
1761 			size = list->size;
1762 			zend_mm_free_heap(heap, list ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1763 			return size;
1764 		}
1765 		prev = list;
1766 		list = list->next;
1767 	}
1768 	ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1769 	return 0;
1770 }
1771 
1772 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1773 {
1774 	zend_mm_huge_list *list = heap->huge_list;
1775 	while (list != NULL) {
1776 		if (list->ptr == ptr) {
1777 			return list->size;
1778 		}
1779 		list = list->next;
1780 	}
1781 	ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1782 	return 0;
1783 }
1784 
1785 #if ZEND_DEBUG
1786 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1787 #else
1788 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1789 #endif
1790 {
1791 	zend_mm_huge_list *list = heap->huge_list;
1792 	while (list != NULL) {
1793 		if (list->ptr == ptr) {
1794 			list->size = size;
1795 #if ZEND_DEBUG
1796 			list->dbg.size = dbg_size;
1797 			list->dbg.filename = __zend_filename;
1798 			list->dbg.orig_filename = __zend_orig_filename;
1799 			list->dbg.lineno = __zend_lineno;
1800 			list->dbg.orig_lineno = __zend_orig_lineno;
1801 #endif
1802 			return;
1803 		}
1804 		list = list->next;
1805 	}
1806 }
1807 
1808 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1809 {
1810 #ifdef ZEND_WIN32
1811 	/* On Windows we don't have ability to extend huge blocks in-place.
1812 	 * We allocate them with 2MB size granularity, to avoid many
1813 	 * reallocations when they are extended by small pieces
1814 	 */
1815 	size_t alignment = MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE);
1816 #else
1817 	size_t alignment = REAL_PAGE_SIZE;
1818 #endif
1819 	size_t new_size = ZEND_MM_ALIGNED_SIZE_EX(size, alignment);
1820 	void *ptr;
1821 
1822 	if (UNEXPECTED(new_size < size)) {
1823 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", size, alignment);
1824 	}
1825 
1826 #if ZEND_MM_LIMIT
1827 	if (UNEXPECTED(new_size > heap->limit - heap->real_size)) {
1828 		if (zend_mm_gc(heap) && new_size <= heap->limit - heap->real_size) {
1829 			/* pass */
1830 		} else if (heap->overflow == 0) {
1831 #if ZEND_DEBUG
1832 			zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1833 #else
1834 			zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1835 #endif
1836 			return NULL;
1837 		}
1838 	}
1839 #endif
1840 	ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE);
1841 	if (UNEXPECTED(ptr == NULL)) {
1842 		/* insufficient memory */
1843 		if (zend_mm_gc(heap) &&
1844 		    (ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE)) != NULL) {
1845 			/* pass */
1846 		} else {
1847 #if !ZEND_MM_LIMIT
1848 			zend_mm_safe_error(heap, "Out of memory");
1849 #elif ZEND_DEBUG
1850 			zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1851 #else
1852 			zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, size);
1853 #endif
1854 			return NULL;
1855 		}
1856 	}
1857 #if ZEND_DEBUG
1858 	zend_mm_add_huge_block(heap, ptr, new_size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1859 #else
1860 	zend_mm_add_huge_block(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1861 #endif
1862 #if ZEND_MM_STAT
1863 	do {
1864 		size_t size = heap->real_size + new_size;
1865 		size_t peak = MAX(heap->real_peak, size);
1866 		heap->real_size = size;
1867 		heap->real_peak = peak;
1868 	} while (0);
1869 	do {
1870 		size_t size = heap->size + new_size;
1871 		size_t peak = MAX(heap->peak, size);
1872 		heap->size = size;
1873 		heap->peak = peak;
1874 	} while (0);
1875 #elif ZEND_MM_LIMIT
1876 	heap->real_size += new_size;
1877 #endif
1878 	return ptr;
1879 }
1880 
1881 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1882 {
1883 	size_t size;
1884 
1885 	ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE) == 0, "zend_mm_heap corrupted");
1886 	size = zend_mm_del_huge_block(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1887 	zend_mm_chunk_free(heap, ptr, size);
1888 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1889 	heap->real_size -= size;
1890 #endif
1891 #if ZEND_MM_STAT
1892 	heap->size -= size;
1893 #endif
1894 }
1895 
1896 /******************/
1897 /* Initialization */
1898 /******************/
1899 
1900 static zend_mm_heap *zend_mm_init(void)
1901 {
1902 	zend_mm_chunk *chunk = (zend_mm_chunk*)zend_mm_chunk_alloc_int(ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1903 	zend_mm_heap *heap;
1904 
1905 	if (UNEXPECTED(chunk == NULL)) {
1906 #if ZEND_MM_ERROR
1907 		fprintf(stderr, "Can't initialize heap\n");
1908 #endif
1909 		return NULL;
1910 	}
1911 	heap = &chunk->heap_slot;
1912 	chunk->heap = heap;
1913 	chunk->next = chunk;
1914 	chunk->prev = chunk;
1915 	chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1916 	chunk->free_tail = ZEND_MM_FIRST_PAGE;
1917 	chunk->num = 0;
1918 	chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
1919 	chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
1920 	heap->main_chunk = chunk;
1921 	heap->cached_chunks = NULL;
1922 	heap->chunks_count = 1;
1923 	heap->peak_chunks_count = 1;
1924 	heap->cached_chunks_count = 0;
1925 	heap->avg_chunks_count = 1.0;
1926 	heap->last_chunks_delete_boundary = 0;
1927 	heap->last_chunks_delete_count = 0;
1928 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1929 	heap->real_size = ZEND_MM_CHUNK_SIZE;
1930 #endif
1931 #if ZEND_MM_STAT
1932 	heap->real_peak = ZEND_MM_CHUNK_SIZE;
1933 	heap->size = 0;
1934 	heap->peak = 0;
1935 #endif
1936 #if ZEND_MM_LIMIT
1937 	heap->limit = (size_t)Z_L(-1) >> 1;
1938 	heap->overflow = 0;
1939 #endif
1940 #if ZEND_MM_CUSTOM
1941 	heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
1942 #endif
1943 #if ZEND_MM_STORAGE
1944 	heap->storage = NULL;
1945 #endif
1946 	heap->huge_list = NULL;
1947 	return heap;
1948 }
1949 
1950 ZEND_API size_t zend_mm_gc(zend_mm_heap *heap)
1951 {
1952 	zend_mm_free_slot *p, **q;
1953 	zend_mm_chunk *chunk;
1954 	size_t page_offset;
1955 	int page_num;
1956 	zend_mm_page_info info;
1957 	uint32_t i, free_counter;
1958 	bool has_free_pages;
1959 	size_t collected = 0;
1960 
1961 #if ZEND_MM_CUSTOM
1962 	if (heap->use_custom_heap) {
1963 		return 0;
1964 	}
1965 #endif
1966 
1967 	for (i = 0; i < ZEND_MM_BINS; i++) {
1968 		has_free_pages = false;
1969 		p = heap->free_slot[i];
1970 		while (p != NULL) {
1971 			chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
1972 			ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1973 			page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
1974 			ZEND_ASSERT(page_offset != 0);
1975 			page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1976 			info = chunk->map[page_num];
1977 			ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1978 			if (info & ZEND_MM_IS_LRUN) {
1979 				page_num -= ZEND_MM_NRUN_OFFSET(info);
1980 				info = chunk->map[page_num];
1981 				ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1982 				ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
1983 			}
1984 			ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
1985 			free_counter = ZEND_MM_SRUN_FREE_COUNTER(info) + 1;
1986 			if (free_counter == bin_elements[i]) {
1987 				has_free_pages = true;
1988 			}
1989 			chunk->map[page_num] = ZEND_MM_SRUN_EX(i, free_counter);
1990 			p = p->next_free_slot;
1991 		}
1992 
1993 		if (!has_free_pages) {
1994 			continue;
1995 		}
1996 
1997 		q = &heap->free_slot[i];
1998 		p = *q;
1999 		while (p != NULL) {
2000 			chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
2001 			ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
2002 			page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
2003 			ZEND_ASSERT(page_offset != 0);
2004 			page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
2005 			info = chunk->map[page_num];
2006 			ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
2007 			if (info & ZEND_MM_IS_LRUN) {
2008 				page_num -= ZEND_MM_NRUN_OFFSET(info);
2009 				info = chunk->map[page_num];
2010 				ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
2011 				ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
2012 			}
2013 			ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
2014 			if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[i]) {
2015 				/* remove from cache */
2016 				p = p->next_free_slot;
2017 				*q = p;
2018 			} else {
2019 				q = &p->next_free_slot;
2020 				p = *q;
2021 			}
2022 		}
2023 	}
2024 
2025 	chunk = heap->main_chunk;
2026 	do {
2027 		i = ZEND_MM_FIRST_PAGE;
2028 		while (i < chunk->free_tail) {
2029 			if (zend_mm_bitset_is_set(chunk->free_map, i)) {
2030 				info = chunk->map[i];
2031 				if (info & ZEND_MM_IS_SRUN) {
2032 					int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
2033 					int pages_count = bin_pages[bin_num];
2034 
2035 					if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[bin_num]) {
2036 						/* all elements are free */
2037 						zend_mm_free_pages_ex(heap, chunk, i, pages_count, 0);
2038 						collected += pages_count;
2039 					} else {
2040 						/* reset counter */
2041 						chunk->map[i] = ZEND_MM_SRUN(bin_num);
2042 					}
2043 					i += bin_pages[bin_num];
2044 				} else /* if (info & ZEND_MM_IS_LRUN) */ {
2045 					i += ZEND_MM_LRUN_PAGES(info);
2046 				}
2047 			} else {
2048 				i++;
2049 			}
2050 		}
2051 		if (chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
2052 			zend_mm_chunk *next_chunk = chunk->next;
2053 
2054 			zend_mm_delete_chunk(heap, chunk);
2055 			chunk = next_chunk;
2056 		} else {
2057 			chunk = chunk->next;
2058 		}
2059 	} while (chunk != heap->main_chunk);
2060 
2061 	return collected * ZEND_MM_PAGE_SIZE;
2062 }
2063 
2064 #if ZEND_DEBUG
2065 /******************/
2066 /* Leak detection */
2067 /******************/
2068 
2069 static zend_long zend_mm_find_leaks_small(zend_mm_chunk *p, uint32_t i, uint32_t j, zend_leak_info *leak)
2070 {
2071 	bool empty = true;
2072 	zend_long count = 0;
2073 	int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2074 	zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * (j + 1) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2075 
2076 	while (j < bin_elements[bin_num]) {
2077 		if (dbg->size != 0) {
2078 			if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2079 				count++;
2080 				dbg->size = 0;
2081 				dbg->filename = NULL;
2082 				dbg->lineno = 0;
2083 			} else {
2084 				empty = false;
2085 			}
2086 		}
2087 		j++;
2088 		dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2089 	}
2090 	if (empty) {
2091 		zend_mm_bitset_reset_range(p->free_map, i, bin_pages[bin_num]);
2092 	}
2093 	return count;
2094 }
2095 
2096 static zend_long zend_mm_find_leaks(zend_mm_heap *heap, zend_mm_chunk *p, uint32_t i, zend_leak_info *leak)
2097 {
2098 	zend_long count = 0;
2099 
2100 	do {
2101 		while (i < p->free_tail) {
2102 			if (zend_mm_bitset_is_set(p->free_map, i)) {
2103 				if (p->map[i] & ZEND_MM_IS_SRUN) {
2104 					int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2105 					count += zend_mm_find_leaks_small(p, i, 0, leak);
2106 					i += bin_pages[bin_num];
2107 				} else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2108 					int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2109 					zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2110 
2111 					if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2112 						count++;
2113 					}
2114 					zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2115 					i += pages_count;
2116 				}
2117 			} else {
2118 				i++;
2119 			}
2120 		}
2121 		p = p->next;
2122 		i = ZEND_MM_FIRST_PAGE;
2123 	} while (p != heap->main_chunk);
2124 	return count;
2125 }
2126 
2127 static zend_long zend_mm_find_leaks_huge(zend_mm_heap *heap, zend_mm_huge_list *list)
2128 {
2129 	zend_long count = 0;
2130 	zend_mm_huge_list *prev = list;
2131 	zend_mm_huge_list *p = list->next;
2132 
2133 	while (p) {
2134 		if (p->dbg.filename == list->dbg.filename && p->dbg.lineno == list->dbg.lineno) {
2135 			prev->next = p->next;
2136 			zend_mm_chunk_free(heap, p->ptr, p->size);
2137 			zend_mm_free_heap(heap, p, NULL, 0, NULL, 0);
2138 			count++;
2139 		} else {
2140 			prev = p;
2141 		}
2142 		p = prev->next;
2143 	}
2144 
2145 	return count;
2146 }
2147 
2148 static void zend_mm_check_leaks(zend_mm_heap *heap)
2149 {
2150 	zend_mm_huge_list *list;
2151 	zend_mm_chunk *p;
2152 	zend_leak_info leak;
2153 	zend_long repeated = 0;
2154 	uint32_t total = 0;
2155 	uint32_t i, j;
2156 
2157 	/* find leaked huge blocks and free them */
2158 	list = heap->huge_list;
2159 	while (list) {
2160 		zend_mm_huge_list *q = list;
2161 
2162 		leak.addr = list->ptr;
2163 		leak.size = list->dbg.size;
2164 		leak.filename = list->dbg.filename;
2165 		leak.orig_filename = list->dbg.orig_filename;
2166 		leak.lineno = list->dbg.lineno;
2167 		leak.orig_lineno = list->dbg.orig_lineno;
2168 
2169 		zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2170 		zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2171 		repeated = zend_mm_find_leaks_huge(heap, list);
2172 		total += 1 + repeated;
2173 		if (repeated) {
2174 			zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2175 		}
2176 
2177 		heap->huge_list = list = list->next;
2178 		zend_mm_chunk_free(heap, q->ptr, q->size);
2179 		zend_mm_free_heap(heap, q, NULL, 0, NULL, 0);
2180 	}
2181 
2182 	/* for each chunk */
2183 	p = heap->main_chunk;
2184 	do {
2185 		i = ZEND_MM_FIRST_PAGE;
2186 		while (i < p->free_tail) {
2187 			if (zend_mm_bitset_is_set(p->free_map, i)) {
2188 				if (p->map[i] & ZEND_MM_IS_SRUN) {
2189 					int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2190 					zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2191 
2192 					j = 0;
2193 					while (j < bin_elements[bin_num]) {
2194 						if (dbg->size != 0) {
2195 							leak.addr = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * j);
2196 							leak.size = dbg->size;
2197 							leak.filename = dbg->filename;
2198 							leak.orig_filename = dbg->orig_filename;
2199 							leak.lineno = dbg->lineno;
2200 							leak.orig_lineno = dbg->orig_lineno;
2201 
2202 							zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2203 							zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2204 
2205 							dbg->size = 0;
2206 							dbg->filename = NULL;
2207 							dbg->lineno = 0;
2208 
2209 							repeated = zend_mm_find_leaks_small(p, i, j + 1, &leak) +
2210 							           zend_mm_find_leaks(heap, p, i + bin_pages[bin_num], &leak);
2211 							total += 1 + repeated;
2212 							if (repeated) {
2213 								zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2214 							}
2215 						}
2216 						dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2217 						j++;
2218 					}
2219 					i += bin_pages[bin_num];
2220 				} else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2221 					int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2222 					zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2223 
2224 					leak.addr = (void*)((char*)p + ZEND_MM_PAGE_SIZE * i);
2225 					leak.size = dbg->size;
2226 					leak.filename = dbg->filename;
2227 					leak.orig_filename = dbg->orig_filename;
2228 					leak.lineno = dbg->lineno;
2229 					leak.orig_lineno = dbg->orig_lineno;
2230 
2231 					zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2232 					zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2233 
2234 					zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2235 
2236 					repeated = zend_mm_find_leaks(heap, p, i + pages_count, &leak);
2237 					total += 1 + repeated;
2238 					if (repeated) {
2239 						zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2240 					}
2241 					i += pages_count;
2242 				}
2243 			} else {
2244 				i++;
2245 			}
2246 		}
2247 		p = p->next;
2248 	} while (p != heap->main_chunk);
2249 	if (total) {
2250 		zend_message_dispatcher(ZMSG_MEMORY_LEAKS_GRAND_TOTAL, &total);
2251 	}
2252 }
2253 #endif
2254 
2255 #if ZEND_MM_CUSTOM
2256 static void *tracked_malloc(size_t size);
2257 static void tracked_free_all(void);
2258 #endif
2259 
2260 void zend_mm_shutdown(zend_mm_heap *heap, bool full, bool silent)
2261 {
2262 	zend_mm_chunk *p;
2263 	zend_mm_huge_list *list;
2264 
2265 #if ZEND_MM_CUSTOM
2266 	if (heap->use_custom_heap) {
2267 		if (heap->custom_heap.std._malloc == tracked_malloc) {
2268 			if (silent) {
2269 				tracked_free_all();
2270 			}
2271 			zend_hash_clean(heap->tracked_allocs);
2272 			if (full) {
2273 				zend_hash_destroy(heap->tracked_allocs);
2274 				free(heap->tracked_allocs);
2275 				/* Make sure the heap free below does not use tracked_free(). */
2276 				heap->custom_heap.std._free = free;
2277 			}
2278 			heap->size = 0;
2279 		}
2280 
2281 		if (full) {
2282 			if (ZEND_DEBUG && heap->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2283 				heap->custom_heap.debug._free(heap ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC);
2284 			} else {
2285 				heap->custom_heap.std._free(heap);
2286 			}
2287 		}
2288 		return;
2289 	}
2290 #endif
2291 
2292 #if ZEND_DEBUG
2293 	if (!silent) {
2294 		zend_mm_check_leaks(heap);
2295 	}
2296 #endif
2297 
2298 	/* free huge blocks */
2299 	list = heap->huge_list;
2300 	heap->huge_list = NULL;
2301 	while (list) {
2302 		zend_mm_huge_list *q = list;
2303 		list = list->next;
2304 		zend_mm_chunk_free(heap, q->ptr, q->size);
2305 	}
2306 
2307 	/* move all chunks except of the first one into the cache */
2308 	p = heap->main_chunk->next;
2309 	while (p != heap->main_chunk) {
2310 		zend_mm_chunk *q = p->next;
2311 		p->next = heap->cached_chunks;
2312 		heap->cached_chunks = p;
2313 		p = q;
2314 		heap->chunks_count--;
2315 		heap->cached_chunks_count++;
2316 	}
2317 
2318 	if (full) {
2319 		/* free all cached chunks */
2320 		while (heap->cached_chunks) {
2321 			p = heap->cached_chunks;
2322 			heap->cached_chunks = p->next;
2323 			zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2324 		}
2325 		/* free the first chunk */
2326 		zend_mm_chunk_free(heap, heap->main_chunk, ZEND_MM_CHUNK_SIZE);
2327 	} else {
2328 		/* free some cached chunks to keep average count */
2329 		heap->avg_chunks_count = (heap->avg_chunks_count + (double)heap->peak_chunks_count) / 2.0;
2330 		while ((double)heap->cached_chunks_count + 0.9 > heap->avg_chunks_count &&
2331 		       heap->cached_chunks) {
2332 			p = heap->cached_chunks;
2333 			heap->cached_chunks = p->next;
2334 			zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2335 			heap->cached_chunks_count--;
2336 		}
2337 		/* clear cached chunks */
2338 		p = heap->cached_chunks;
2339 		while (p != NULL) {
2340 			zend_mm_chunk *q = p->next;
2341 			memset(p, 0, sizeof(zend_mm_chunk));
2342 			p->next = q;
2343 			p = q;
2344 		}
2345 
2346 		/* reinitialize the first chunk and heap */
2347 		p = heap->main_chunk;
2348 		p->heap = &p->heap_slot;
2349 		p->next = p;
2350 		p->prev = p;
2351 		p->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
2352 		p->free_tail = ZEND_MM_FIRST_PAGE;
2353 		p->num = 0;
2354 
2355 #if ZEND_MM_STAT
2356 		heap->size = heap->peak = 0;
2357 #endif
2358 		memset(heap->free_slot, 0, sizeof(heap->free_slot));
2359 #if ZEND_MM_STAT || ZEND_MM_LIMIT
2360 		heap->real_size = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE;
2361 #endif
2362 #if ZEND_MM_STAT
2363 		heap->real_peak = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE;
2364 #endif
2365 		heap->chunks_count = 1;
2366 		heap->peak_chunks_count = 1;
2367 		heap->last_chunks_delete_boundary = 0;
2368 		heap->last_chunks_delete_count = 0;
2369 
2370 		memset(p->free_map, 0, sizeof(p->free_map) + sizeof(p->map));
2371 		p->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
2372 		p->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
2373 	}
2374 }
2375 
2376 /**************/
2377 /* PUBLIC API */
2378 /**************/
2379 
2380 ZEND_API void* ZEND_FASTCALL _zend_mm_alloc(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2381 {
2382 	return zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2383 }
2384 
2385 ZEND_API void ZEND_FASTCALL _zend_mm_free(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2386 {
2387 	zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2388 }
2389 
2390 void* ZEND_FASTCALL _zend_mm_realloc(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2391 {
2392 	return zend_mm_realloc_heap(heap, ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2393 }
2394 
2395 void* ZEND_FASTCALL _zend_mm_realloc2(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2396 {
2397 	return zend_mm_realloc_heap(heap, ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2398 }
2399 
2400 ZEND_API size_t ZEND_FASTCALL _zend_mm_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2401 {
2402 	return zend_mm_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2403 }
2404 
2405 /**********************/
2406 /* Allocation Manager */
2407 /**********************/
2408 
2409 typedef struct _zend_alloc_globals {
2410 	zend_mm_heap *mm_heap;
2411 } zend_alloc_globals;
2412 
2413 #ifdef ZTS
2414 static int alloc_globals_id;
2415 static size_t alloc_globals_offset;
2416 # define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v)
2417 #else
2418 # define AG(v) (alloc_globals.v)
2419 static zend_alloc_globals alloc_globals;
2420 #endif
2421 
2422 ZEND_API bool is_zend_mm(void)
2423 {
2424 #if ZEND_MM_CUSTOM
2425 	return !AG(mm_heap)->use_custom_heap;
2426 #else
2427 	return 1;
2428 #endif
2429 }
2430 
2431 ZEND_API bool is_zend_ptr(const void *ptr)
2432 {
2433 #if ZEND_MM_CUSTOM
2434 	if (AG(mm_heap)->use_custom_heap) {
2435 		if (AG(mm_heap)->custom_heap.std._malloc == tracked_malloc) {
2436 			zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2437 			zval *size_zv = zend_hash_index_find(AG(mm_heap)->tracked_allocs, h);
2438 			if  (size_zv) {
2439 				return 1;
2440 			}
2441 		}
2442 		return 0;
2443 	}
2444 #endif
2445 
2446 	if (AG(mm_heap)->main_chunk) {
2447 		zend_mm_chunk *chunk = AG(mm_heap)->main_chunk;
2448 
2449 		do {
2450 			if (ptr >= (void*)chunk
2451 			 && ptr < (void*)((char*)chunk + ZEND_MM_CHUNK_SIZE)) {
2452 				return 1;
2453 			}
2454 			chunk = chunk->next;
2455 		} while (chunk != AG(mm_heap)->main_chunk);
2456 	}
2457 
2458 	zend_mm_huge_list *block = AG(mm_heap)->huge_list;
2459 	while (block) {
2460 		if (ptr >= (void*)block
2461 				&& ptr < (void*)((char*)block + block->size)) {
2462 			return 1;
2463 		}
2464 		block = block->next;
2465 	}
2466 
2467 	return 0;
2468 }
2469 
2470 #if ZEND_MM_CUSTOM
2471 
2472 static ZEND_COLD void* ZEND_FASTCALL _malloc_custom(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2473 {
2474 	if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2475 		return AG(mm_heap)->custom_heap.debug._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2476 	} else {
2477 		return AG(mm_heap)->custom_heap.std._malloc(size);
2478 	}
2479 }
2480 
2481 static ZEND_COLD void ZEND_FASTCALL _efree_custom(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2482 {
2483 	if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2484 		AG(mm_heap)->custom_heap.debug._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2485 	} else {
2486 		AG(mm_heap)->custom_heap.std._free(ptr);
2487 	}
2488 }
2489 
2490 static ZEND_COLD void* ZEND_FASTCALL _realloc_custom(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2491 {
2492 	if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2493 		return AG(mm_heap)->custom_heap.debug._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2494 	} else {
2495 		return AG(mm_heap)->custom_heap.std._realloc(ptr, size);
2496 	}
2497 }
2498 #endif
2499 
2500 #if !ZEND_DEBUG && defined(HAVE_BUILTIN_CONSTANT_P)
2501 #undef _emalloc
2502 
2503 #if ZEND_MM_CUSTOM
2504 # define ZEND_MM_CUSTOM_ALLOCATOR(size) do { \
2505 		if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2506 			return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2507 		} \
2508 	} while (0)
2509 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr) do { \
2510 		if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2511 			_efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2512 			return; \
2513 		} \
2514 	} while (0)
2515 #else
2516 # define ZEND_MM_CUSTOM_ALLOCATOR(size)
2517 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr)
2518 #endif
2519 
2520 # define _ZEND_BIN_ALLOCATOR(_num, _size, _elements, _pages, x, y) \
2521 	ZEND_API void* ZEND_FASTCALL _emalloc_ ## _size(void) { \
2522 		ZEND_MM_CUSTOM_ALLOCATOR(_size); \
2523 		return zend_mm_alloc_small(AG(mm_heap), _num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2524 	}
2525 
2526 ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR, x, y)
2527 
2528 ZEND_API void* ZEND_FASTCALL _emalloc_large(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2529 {
2530 	ZEND_MM_CUSTOM_ALLOCATOR(size);
2531 	return zend_mm_alloc_large_ex(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2532 }
2533 
2534 ZEND_API void* ZEND_FASTCALL _emalloc_huge(size_t size)
2535 {
2536 	ZEND_MM_CUSTOM_ALLOCATOR(size);
2537 	return zend_mm_alloc_huge(AG(mm_heap), size);
2538 }
2539 
2540 #if ZEND_DEBUG
2541 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2542 	ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2543 		ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2544 		{ \
2545 			size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); \
2546 			zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2547 			int page_num = page_offset / ZEND_MM_PAGE_SIZE; \
2548 			ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2549 			ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_SRUN); \
2550 			ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(chunk->map[page_num]) == _num); \
2551 			zend_mm_free_small(AG(mm_heap), ptr, _num); \
2552 		} \
2553 	}
2554 #else
2555 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2556 	ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2557 		ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2558 		{ \
2559 			zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2560 			ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2561 			zend_mm_free_small(AG(mm_heap), ptr, _num); \
2562 		} \
2563 	}
2564 #endif
2565 
2566 ZEND_MM_BINS_INFO(_ZEND_BIN_FREE, x, y)
2567 
2568 ZEND_API void ZEND_FASTCALL _efree_large(void *ptr, size_t size)
2569 {
2570 	ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2571 	{
2572 		size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
2573 		zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
2574 		int page_num = page_offset / ZEND_MM_PAGE_SIZE;
2575 		uint32_t pages_count = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE) / ZEND_MM_PAGE_SIZE;
2576 
2577 		ZEND_MM_CHECK(chunk->heap == AG(mm_heap) && ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
2578 		ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_LRUN);
2579 		ZEND_ASSERT(ZEND_MM_LRUN_PAGES(chunk->map[page_num]) == pages_count);
2580 		zend_mm_free_large(AG(mm_heap), chunk, page_num, pages_count);
2581 	}
2582 }
2583 
2584 ZEND_API void ZEND_FASTCALL _efree_huge(void *ptr, size_t size)
2585 {
2586 
2587 	ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2588 	zend_mm_free_huge(AG(mm_heap), ptr);
2589 }
2590 #endif
2591 
2592 ZEND_API void* ZEND_FASTCALL _emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2593 {
2594 #if ZEND_MM_CUSTOM
2595 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2596 		return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2597 	}
2598 #endif
2599 	return zend_mm_alloc_heap(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2600 }
2601 
2602 ZEND_API void ZEND_FASTCALL _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2603 {
2604 #if ZEND_MM_CUSTOM
2605 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2606 		_efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2607 		return;
2608 	}
2609 #endif
2610 	zend_mm_free_heap(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2611 }
2612 
2613 ZEND_API void* ZEND_FASTCALL _erealloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2614 {
2615 #if ZEND_MM_CUSTOM
2616 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2617 		return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2618 	}
2619 #endif
2620 	return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2621 }
2622 
2623 ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2624 {
2625 #if ZEND_MM_CUSTOM
2626 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2627 		return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2628 	}
2629 #endif
2630 	return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2631 }
2632 
2633 ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2634 {
2635 #if ZEND_MM_CUSTOM
2636 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2637 		return 0;
2638 	}
2639 #endif
2640 	return zend_mm_size(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2641 }
2642 
2643 ZEND_API void* ZEND_FASTCALL _safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2644 {
2645 	return _emalloc(zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2646 }
2647 
2648 ZEND_API void* ZEND_FASTCALL _safe_malloc(size_t nmemb, size_t size, size_t offset)
2649 {
2650 	return pemalloc(zend_safe_address_guarded(nmemb, size, offset), 1);
2651 }
2652 
2653 ZEND_API void* ZEND_FASTCALL _safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2654 {
2655 	return _erealloc(ptr, zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2656 }
2657 
2658 ZEND_API void* ZEND_FASTCALL _safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset)
2659 {
2660 	return perealloc(ptr, zend_safe_address_guarded(nmemb, size, offset), 1);
2661 }
2662 
2663 ZEND_API void* ZEND_FASTCALL _ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2664 {
2665 	void *p;
2666 
2667 	size = zend_safe_address_guarded(nmemb, size, 0);
2668 	p = _emalloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2669 	memset(p, 0, size);
2670 	return p;
2671 }
2672 
2673 ZEND_API char* ZEND_FASTCALL _estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2674 {
2675 	size_t length;
2676 	char *p;
2677 
2678 	length = strlen(s);
2679 	if (UNEXPECTED(length + 1 == 0)) {
2680 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2681 	}
2682 	p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2683 	memcpy(p, s, length+1);
2684 	return p;
2685 }
2686 
2687 ZEND_API char* ZEND_FASTCALL _estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2688 {
2689 	char *p;
2690 
2691 	if (UNEXPECTED(length + 1 == 0)) {
2692 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2693 	}
2694 	p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2695 	memcpy(p, s, length);
2696 	p[length] = 0;
2697 	return p;
2698 }
2699 
2700 static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void);
2701 
2702 ZEND_API char* ZEND_FASTCALL zend_strndup(const char *s, size_t length)
2703 {
2704 	char *p;
2705 
2706 	if (UNEXPECTED(length + 1 == 0)) {
2707 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2708 	}
2709 	p = (char *) malloc(length + 1);
2710 	if (UNEXPECTED(p == NULL)) {
2711 		zend_out_of_memory();
2712 	}
2713 	if (EXPECTED(length)) {
2714 		memcpy(p, s, length);
2715 	}
2716 	p[length] = 0;
2717 	return p;
2718 }
2719 
2720 ZEND_API zend_result zend_set_memory_limit(size_t memory_limit)
2721 {
2722 #if ZEND_MM_LIMIT
2723 	zend_mm_heap *heap = AG(mm_heap);
2724 
2725 	if (UNEXPECTED(memory_limit < heap->real_size)) {
2726 		if (memory_limit >= heap->real_size - heap->cached_chunks_count * ZEND_MM_CHUNK_SIZE) {
2727 			/* free some cached chunks to fit into new memory limit */
2728 			do {
2729 				zend_mm_chunk *p = heap->cached_chunks;
2730 				heap->cached_chunks = p->next;
2731 				zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2732 				heap->cached_chunks_count--;
2733 				heap->real_size -= ZEND_MM_CHUNK_SIZE;
2734 			} while (memory_limit < heap->real_size);
2735 			return SUCCESS;
2736 		}
2737 		return FAILURE;
2738 	}
2739 	AG(mm_heap)->limit = memory_limit;
2740 #endif
2741 	return SUCCESS;
2742 }
2743 
2744 ZEND_API bool zend_alloc_in_memory_limit_error_reporting(void)
2745 {
2746 #if ZEND_MM_LIMIT
2747 	return AG(mm_heap)->overflow;
2748 #else
2749 	return false;
2750 #endif
2751 }
2752 
2753 ZEND_API size_t zend_memory_usage(bool real_usage)
2754 {
2755 #if ZEND_MM_STAT
2756 	if (real_usage) {
2757 		return AG(mm_heap)->real_size;
2758 	} else {
2759 		size_t usage = AG(mm_heap)->size;
2760 		return usage;
2761 	}
2762 #endif
2763 	return 0;
2764 }
2765 
2766 ZEND_API size_t zend_memory_peak_usage(bool real_usage)
2767 {
2768 #if ZEND_MM_STAT
2769 	if (real_usage) {
2770 		return AG(mm_heap)->real_peak;
2771 	} else {
2772 		return AG(mm_heap)->peak;
2773 	}
2774 #endif
2775 	return 0;
2776 }
2777 
2778 ZEND_API void zend_memory_reset_peak_usage(void)
2779 {
2780 #if ZEND_MM_STAT
2781 	AG(mm_heap)->real_peak = AG(mm_heap)->real_size;
2782 	AG(mm_heap)->peak = AG(mm_heap)->size;
2783 #endif
2784 }
2785 
2786 ZEND_API void shutdown_memory_manager(bool silent, bool full_shutdown)
2787 {
2788 	zend_mm_shutdown(AG(mm_heap), full_shutdown, silent);
2789 }
2790 
2791 static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void)
2792 {
2793 	fprintf(stderr, "Out of memory\n");
2794 	exit(1);
2795 }
2796 
2797 #if ZEND_MM_CUSTOM
2798 static zend_always_inline void tracked_add(zend_mm_heap *heap, void *ptr, size_t size) {
2799 	zval size_zv;
2800 	zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2801 	ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr);
2802 	ZVAL_LONG(&size_zv, size);
2803 	zend_hash_index_add_new(heap->tracked_allocs, h, &size_zv);
2804 }
2805 
2806 static zend_always_inline zval *tracked_get_size_zv(zend_mm_heap *heap, void *ptr) {
2807 	zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2808 	zval *size_zv = zend_hash_index_find(heap->tracked_allocs, h);
2809 	ZEND_ASSERT(size_zv && "Trying to free pointer not allocated through ZendMM");
2810 	return size_zv;
2811 }
2812 
2813 static zend_always_inline void tracked_check_limit(zend_mm_heap *heap, size_t add_size) {
2814 	if (add_size > heap->limit - heap->size && !heap->overflow) {
2815 #if ZEND_DEBUG
2816 		zend_mm_safe_error(heap,
2817 			"Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)",
2818 			heap->limit, "file", 0, add_size);
2819 #else
2820 		zend_mm_safe_error(heap,
2821 			"Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)",
2822 			heap->limit, add_size);
2823 #endif
2824 	}
2825 }
2826 
2827 static void *tracked_malloc(size_t size)
2828 {
2829 	zend_mm_heap *heap = AG(mm_heap);
2830 	tracked_check_limit(heap, size);
2831 
2832 	void *ptr = malloc(size);
2833 	if (!ptr) {
2834 		zend_out_of_memory();
2835 	}
2836 
2837 	tracked_add(heap, ptr, size);
2838 	heap->size += size;
2839 	return ptr;
2840 }
2841 
2842 static void tracked_free(void *ptr) {
2843 	if (!ptr) {
2844 		return;
2845 	}
2846 
2847 	zend_mm_heap *heap = AG(mm_heap);
2848 	zval *size_zv = tracked_get_size_zv(heap, ptr);
2849 	heap->size -= Z_LVAL_P(size_zv);
2850 	zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) size_zv);
2851 	free(ptr);
2852 }
2853 
2854 static void *tracked_realloc(void *ptr, size_t new_size) {
2855 	zend_mm_heap *heap = AG(mm_heap);
2856 	zval *old_size_zv = NULL;
2857 	size_t old_size = 0;
2858 	if (ptr) {
2859 		old_size_zv = tracked_get_size_zv(heap, ptr);
2860 		old_size = Z_LVAL_P(old_size_zv);
2861 	}
2862 
2863 	if (new_size > old_size) {
2864 		tracked_check_limit(heap, new_size - old_size);
2865 	}
2866 
2867 	/* Delete information about old allocation only after checking the memory limit. */
2868 	if (old_size_zv) {
2869 		zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) old_size_zv);
2870 	}
2871 
2872 	ptr = __zend_realloc(ptr, new_size);
2873 	tracked_add(heap, ptr, new_size);
2874 	heap->size += new_size - old_size;
2875 	return ptr;
2876 }
2877 
2878 static void tracked_free_all(void) {
2879 	HashTable *tracked_allocs = AG(mm_heap)->tracked_allocs;
2880 	zend_ulong h;
2881 	ZEND_HASH_FOREACH_NUM_KEY(tracked_allocs, h) {
2882 		void *ptr = (void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2);
2883 		free(ptr);
2884 	} ZEND_HASH_FOREACH_END();
2885 }
2886 #endif
2887 
2888 static void alloc_globals_ctor(zend_alloc_globals *alloc_globals)
2889 {
2890 	char *tmp;
2891 
2892 #if ZEND_MM_CUSTOM
2893 	tmp = getenv("USE_ZEND_ALLOC");
2894 	if (tmp && !ZEND_ATOL(tmp)) {
2895 		bool tracked = (tmp = getenv("USE_TRACKED_ALLOC")) && ZEND_ATOL(tmp);
2896 		zend_mm_heap *mm_heap = alloc_globals->mm_heap = malloc(sizeof(zend_mm_heap));
2897 		memset(mm_heap, 0, sizeof(zend_mm_heap));
2898 		mm_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2899 		mm_heap->limit = (size_t)Z_L(-1) >> 1;
2900 		mm_heap->overflow = 0;
2901 
2902 		if (!tracked) {
2903 			/* Use system allocator. */
2904 			mm_heap->custom_heap.std._malloc = __zend_malloc;
2905 			mm_heap->custom_heap.std._free = free;
2906 			mm_heap->custom_heap.std._realloc = __zend_realloc;
2907 		} else {
2908 			/* Use system allocator and track allocations for auto-free. */
2909 			mm_heap->custom_heap.std._malloc = tracked_malloc;
2910 			mm_heap->custom_heap.std._free = tracked_free;
2911 			mm_heap->custom_heap.std._realloc = tracked_realloc;
2912 			mm_heap->tracked_allocs = malloc(sizeof(HashTable));
2913 			zend_hash_init(mm_heap->tracked_allocs, 1024, NULL, NULL, 1);
2914 		}
2915 		return;
2916 	}
2917 #endif
2918 
2919 	tmp = getenv("USE_ZEND_ALLOC_HUGE_PAGES");
2920 	if (tmp && ZEND_ATOL(tmp)) {
2921 		zend_mm_use_huge_pages = true;
2922 	}
2923 	alloc_globals->mm_heap = zend_mm_init();
2924 }
2925 
2926 #ifdef ZTS
2927 static void alloc_globals_dtor(zend_alloc_globals *alloc_globals)
2928 {
2929 	zend_mm_shutdown(alloc_globals->mm_heap, 1, 1);
2930 }
2931 #endif
2932 
2933 ZEND_API void start_memory_manager(void)
2934 {
2935 #ifdef ZTS
2936 	ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
2937 #else
2938 	alloc_globals_ctor(&alloc_globals);
2939 #endif
2940 #ifndef _WIN32
2941 #  if defined(_SC_PAGESIZE)
2942 	REAL_PAGE_SIZE = sysconf(_SC_PAGESIZE);
2943 #  elif defined(_SC_PAGE_SIZE)
2944 	REAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
2945 #  endif
2946 #endif
2947 }
2948 
2949 ZEND_API zend_mm_heap *zend_mm_set_heap(zend_mm_heap *new_heap)
2950 {
2951 	zend_mm_heap *old_heap;
2952 
2953 	old_heap = AG(mm_heap);
2954 	AG(mm_heap) = (zend_mm_heap*)new_heap;
2955 	return (zend_mm_heap*)old_heap;
2956 }
2957 
2958 ZEND_API zend_mm_heap *zend_mm_get_heap(void)
2959 {
2960 	return AG(mm_heap);
2961 }
2962 
2963 ZEND_API bool zend_mm_is_custom_heap(zend_mm_heap *new_heap)
2964 {
2965 #if ZEND_MM_CUSTOM
2966 	return AG(mm_heap)->use_custom_heap;
2967 #else
2968 	return 0;
2969 #endif
2970 }
2971 
2972 ZEND_API void zend_mm_set_custom_handlers(zend_mm_heap *heap,
2973                                           void* (*_malloc)(size_t),
2974                                           void  (*_free)(void*),
2975                                           void* (*_realloc)(void*, size_t))
2976 {
2977 #if ZEND_MM_CUSTOM
2978 	zend_mm_heap *_heap = (zend_mm_heap*)heap;
2979 
2980 	if (!_malloc && !_free && !_realloc) {
2981 		_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
2982 	} else {
2983 		_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2984 		_heap->custom_heap.std._malloc = _malloc;
2985 		_heap->custom_heap.std._free = _free;
2986 		_heap->custom_heap.std._realloc = _realloc;
2987 	}
2988 #endif
2989 }
2990 
2991 ZEND_API void zend_mm_get_custom_handlers(zend_mm_heap *heap,
2992                                           void* (**_malloc)(size_t),
2993                                           void  (**_free)(void*),
2994                                           void* (**_realloc)(void*, size_t))
2995 {
2996 #if ZEND_MM_CUSTOM
2997 	zend_mm_heap *_heap = (zend_mm_heap*)heap;
2998 
2999 	if (heap->use_custom_heap) {
3000 		*_malloc = _heap->custom_heap.std._malloc;
3001 		*_free = _heap->custom_heap.std._free;
3002 		*_realloc = _heap->custom_heap.std._realloc;
3003 	} else {
3004 		*_malloc = NULL;
3005 		*_free = NULL;
3006 		*_realloc = NULL;
3007 	}
3008 #else
3009 	*_malloc = NULL;
3010 	*_free = NULL;
3011 	*_realloc = NULL;
3012 #endif
3013 }
3014 
3015 #if ZEND_DEBUG
3016 ZEND_API void zend_mm_set_custom_debug_handlers(zend_mm_heap *heap,
3017                                                 void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
3018                                                 void  (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
3019                                                 void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC))
3020 {
3021 #if ZEND_MM_CUSTOM
3022 	zend_mm_heap *_heap = (zend_mm_heap*)heap;
3023 
3024 	_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_DEBUG;
3025 	_heap->custom_heap.debug._malloc = _malloc;
3026 	_heap->custom_heap.debug._free = _free;
3027 	_heap->custom_heap.debug._realloc = _realloc;
3028 #endif
3029 }
3030 #endif
3031 
3032 ZEND_API zend_mm_storage *zend_mm_get_storage(zend_mm_heap *heap)
3033 {
3034 #if ZEND_MM_STORAGE
3035 	return heap->storage;
3036 #else
3037 	return NULL
3038 #endif
3039 }
3040 
3041 ZEND_API zend_mm_heap *zend_mm_startup(void)
3042 {
3043 	return zend_mm_init();
3044 }
3045 
3046 ZEND_API zend_mm_heap *zend_mm_startup_ex(const zend_mm_handlers *handlers, void *data, size_t data_size)
3047 {
3048 #if ZEND_MM_STORAGE
3049 	zend_mm_storage tmp_storage, *storage;
3050 	zend_mm_chunk *chunk;
3051 	zend_mm_heap *heap;
3052 
3053 	memcpy((zend_mm_handlers*)&tmp_storage.handlers, handlers, sizeof(zend_mm_handlers));
3054 	tmp_storage.data = data;
3055 	chunk = (zend_mm_chunk*)handlers->chunk_alloc(&tmp_storage, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
3056 	if (UNEXPECTED(chunk == NULL)) {
3057 #if ZEND_MM_ERROR
3058 		fprintf(stderr, "Can't initialize heap\n");
3059 #endif
3060 		return NULL;
3061 	}
3062 	heap = &chunk->heap_slot;
3063 	chunk->heap = heap;
3064 	chunk->next = chunk;
3065 	chunk->prev = chunk;
3066 	chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
3067 	chunk->free_tail = ZEND_MM_FIRST_PAGE;
3068 	chunk->num = 0;
3069 	chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
3070 	chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
3071 	heap->main_chunk = chunk;
3072 	heap->cached_chunks = NULL;
3073 	heap->chunks_count = 1;
3074 	heap->peak_chunks_count = 1;
3075 	heap->cached_chunks_count = 0;
3076 	heap->avg_chunks_count = 1.0;
3077 	heap->last_chunks_delete_boundary = 0;
3078 	heap->last_chunks_delete_count = 0;
3079 #if ZEND_MM_STAT || ZEND_MM_LIMIT
3080 	heap->real_size = ZEND_MM_CHUNK_SIZE;
3081 #endif
3082 #if ZEND_MM_STAT
3083 	heap->real_peak = ZEND_MM_CHUNK_SIZE;
3084 	heap->size = 0;
3085 	heap->peak = 0;
3086 #endif
3087 #if ZEND_MM_LIMIT
3088 	heap->limit = (size_t)Z_L(-1) >> 1;
3089 	heap->overflow = 0;
3090 #endif
3091 #if ZEND_MM_CUSTOM
3092 	heap->use_custom_heap = 0;
3093 #endif
3094 	heap->storage = &tmp_storage;
3095 	heap->huge_list = NULL;
3096 	memset(heap->free_slot, 0, sizeof(heap->free_slot));
3097 	storage = _zend_mm_alloc(heap, sizeof(zend_mm_storage) + data_size ZEND_FILE_LINE_CC ZEND_FILE_LINE_CC);
3098 	if (!storage) {
3099 		handlers->chunk_free(&tmp_storage, chunk, ZEND_MM_CHUNK_SIZE);
3100 #if ZEND_MM_ERROR
3101 		fprintf(stderr, "Can't initialize heap\n");
3102 #endif
3103 		return NULL;
3104 	}
3105 	memcpy(storage, &tmp_storage, sizeof(zend_mm_storage));
3106 	if (data) {
3107 		storage->data = (void*)(((char*)storage + sizeof(zend_mm_storage)));
3108 		memcpy(storage->data, data, data_size);
3109 	}
3110 	heap->storage = storage;
3111 	return heap;
3112 #else
3113 	return NULL;
3114 #endif
3115 }
3116 
3117 ZEND_API void * __zend_malloc(size_t len)
3118 {
3119 	void *tmp = malloc(len);
3120 	if (EXPECTED(tmp || !len)) {
3121 		return tmp;
3122 	}
3123 	zend_out_of_memory();
3124 }
3125 
3126 ZEND_API void * __zend_calloc(size_t nmemb, size_t len)
3127 {
3128 	void *tmp;
3129 
3130 	len = zend_safe_address_guarded(nmemb, len, 0);
3131 	tmp = __zend_malloc(len);
3132 	memset(tmp, 0, len);
3133 	return tmp;
3134 }
3135 
3136 ZEND_API void * __zend_realloc(void *p, size_t len)
3137 {
3138 	p = realloc(p, len);
3139 	if (EXPECTED(p || !len)) {
3140 		return p;
3141 	}
3142 	zend_out_of_memory();
3143 }
3144 
3145 ZEND_API char * __zend_strdup(const char *s)
3146 {
3147 	char *tmp = strdup(s);
3148 	if (EXPECTED(tmp)) {
3149 		return tmp;
3150 	}
3151 	zend_out_of_memory();
3152 }
3153 
3154 #ifdef ZTS
3155 size_t zend_mm_globals_size(void)
3156 {
3157 	return sizeof(zend_alloc_globals);
3158 }
3159 #endif
3160