xref: /PHP-7.4/Zend/zend_alloc.c (revision 600402d9)
1 /*
2    +----------------------------------------------------------------------+
3    | Zend Engine                                                          |
4    +----------------------------------------------------------------------+
5    | Copyright (c) Zend Technologies Ltd. (http://www.zend.com)           |
6    +----------------------------------------------------------------------+
7    | This source file is subject to version 2.00 of the Zend license,     |
8    | that is bundled with this package in the file LICENSE, and is        |
9    | available through the world-wide-web at the following url:           |
10    | http://www.zend.com/license/2_00.txt.                                |
11    | If you did not receive a copy of the Zend license and are unable to  |
12    | obtain it through the world-wide-web, please send a note to          |
13    | license@zend.com so we can mail you a copy immediately.              |
14    +----------------------------------------------------------------------+
15    | Authors: Andi Gutmans <andi@php.net>                                 |
16    |          Zeev Suraski <zeev@php.net>                                 |
17    |          Dmitry Stogov <dmitry@php.net>                              |
18    +----------------------------------------------------------------------+
19 */
20 
21 /*
22  * zend_alloc is designed to be a modern CPU cache friendly memory manager
23  * for PHP. Most ideas are taken from jemalloc and tcmalloc implementations.
24  *
25  * All allocations are split into 3 categories:
26  *
27  * Huge  - the size is greater than CHUNK size (~2M by default), allocation is
28  *         performed using mmap(). The result is aligned on 2M boundary.
29  *
30  * Large - a number of 4096K pages inside a CHUNK. Large blocks
31  *         are always aligned on page boundary.
32  *
33  * Small - less than 3/4 of page size. Small sizes are rounded up to nearest
34  *         greater predefined small size (there are 30 predefined sizes:
35  *         8, 16, 24, 32, ... 3072). Small blocks are allocated from
36  *         RUNs. Each RUN is allocated as a single or few following pages.
37  *         Allocation inside RUNs implemented using linked list of free
38  *         elements. The result is aligned to 8 bytes.
39  *
40  * zend_alloc allocates memory from OS by CHUNKs, these CHUNKs and huge memory
41  * blocks are always aligned to CHUNK boundary. So it's very easy to determine
42  * the CHUNK owning the certain pointer. Regular CHUNKs reserve a single
43  * page at start for special purpose. It contains bitset of free pages,
44  * few bitset for available runs of predefined small sizes, map of pages that
45  * keeps information about usage of each page in this CHUNK, etc.
46  *
47  * zend_alloc provides familiar emalloc/efree/erealloc API, but in addition it
48  * provides specialized and optimized routines to allocate blocks of predefined
49  * sizes (e.g. emalloc_2(), emallc_4(), ..., emalloc_large(), etc)
50  * The library uses C preprocessor tricks that substitute calls to emalloc()
51  * with more specialized routines when the requested size is known.
52  */
53 
54 #include "zend.h"
55 #include "zend_alloc.h"
56 #include "zend_globals.h"
57 #include "zend_operators.h"
58 #include "zend_multiply.h"
59 #include "zend_bitset.h"
60 #include <signal.h>
61 
62 #ifdef HAVE_UNISTD_H
63 # include <unistd.h>
64 #endif
65 
66 #ifdef ZEND_WIN32
67 # include <wincrypt.h>
68 # include <process.h>
69 # include "win32/winutil.h"
70 #endif
71 
72 #include <stdio.h>
73 #include <stdlib.h>
74 #include <string.h>
75 
76 #include <sys/types.h>
77 #include <sys/stat.h>
78 #include <limits.h>
79 #include <fcntl.h>
80 #include <errno.h>
81 
82 #ifndef _WIN32
83 # include <sys/mman.h>
84 # ifndef MAP_ANON
85 #  ifdef MAP_ANONYMOUS
86 #   define MAP_ANON MAP_ANONYMOUS
87 #  endif
88 # endif
89 # ifndef MAP_FAILED
90 #  define MAP_FAILED ((void*)-1)
91 # endif
92 # ifndef MAP_POPULATE
93 #  define MAP_POPULATE 0
94 # endif
95 #  if defined(_SC_PAGESIZE) || (_SC_PAGE_SIZE)
96 #    define REAL_PAGE_SIZE _real_page_size
97 static size_t _real_page_size = ZEND_MM_PAGE_SIZE;
98 #  endif
99 # ifdef MAP_ALIGNED_SUPER
100 #    define MAP_HUGETLB MAP_ALIGNED_SUPER
101 # endif
102 #endif
103 
104 #ifndef REAL_PAGE_SIZE
105 # define REAL_PAGE_SIZE ZEND_MM_PAGE_SIZE
106 #endif
107 
108 /* NetBSD has an mremap() function with a signature that is incompatible with Linux (WTF?),
109  * so pretend it doesn't exist. */
110 #ifndef __linux__
111 # undef HAVE_MREMAP
112 #endif
113 
114 #ifndef __APPLE__
115 # define ZEND_MM_FD -1
116 #else
117 /* Mac allows to track anonymous page via vmmap per TAG id.
118  * user land applications are allowed to take from 240 to 255.
119  */
120 # define ZEND_MM_FD (250<<24)
121 #endif
122 
123 #ifndef ZEND_MM_STAT
124 # define ZEND_MM_STAT 1    /* track current and peak memory usage            */
125 #endif
126 #ifndef ZEND_MM_LIMIT
127 # define ZEND_MM_LIMIT 1   /* support for user-defined memory limit          */
128 #endif
129 #ifndef ZEND_MM_CUSTOM
130 # define ZEND_MM_CUSTOM 1  /* support for custom memory allocator            */
131                            /* USE_ZEND_ALLOC=0 may switch to system malloc() */
132 #endif
133 #ifndef ZEND_MM_STORAGE
134 # define ZEND_MM_STORAGE 1 /* support for custom memory storage              */
135 #endif
136 #ifndef ZEND_MM_ERROR
137 # define ZEND_MM_ERROR 1   /* report system errors                           */
138 #endif
139 
140 #ifndef ZEND_MM_CHECK
141 # define ZEND_MM_CHECK(condition, message)  do { \
142 		if (UNEXPECTED(!(condition))) { \
143 			zend_mm_panic(message); \
144 		} \
145 	} while (0)
146 #endif
147 
148 typedef uint32_t   zend_mm_page_info; /* 4-byte integer */
149 typedef zend_ulong zend_mm_bitset;    /* 4-byte or 8-byte integer */
150 
151 #define ZEND_MM_ALIGNED_OFFSET(size, alignment) \
152 	(((size_t)(size)) & ((alignment) - 1))
153 #define ZEND_MM_ALIGNED_BASE(size, alignment) \
154 	(((size_t)(size)) & ~((alignment) - 1))
155 #define ZEND_MM_SIZE_TO_NUM(size, alignment) \
156 	(((size_t)(size) + ((alignment) - 1)) / (alignment))
157 
158 #define ZEND_MM_BITSET_LEN		(sizeof(zend_mm_bitset) * 8)       /* 32 or 64 */
159 #define ZEND_MM_PAGE_MAP_LEN	(ZEND_MM_PAGES / ZEND_MM_BITSET_LEN) /* 16 or 8 */
160 
161 typedef zend_mm_bitset zend_mm_page_map[ZEND_MM_PAGE_MAP_LEN];     /* 64B */
162 
163 #define ZEND_MM_IS_FRUN                  0x00000000
164 #define ZEND_MM_IS_LRUN                  0x40000000
165 #define ZEND_MM_IS_SRUN                  0x80000000
166 
167 #define ZEND_MM_LRUN_PAGES_MASK          0x000003ff
168 #define ZEND_MM_LRUN_PAGES_OFFSET        0
169 
170 #define ZEND_MM_SRUN_BIN_NUM_MASK        0x0000001f
171 #define ZEND_MM_SRUN_BIN_NUM_OFFSET      0
172 
173 #define ZEND_MM_SRUN_FREE_COUNTER_MASK   0x01ff0000
174 #define ZEND_MM_SRUN_FREE_COUNTER_OFFSET 16
175 
176 #define ZEND_MM_NRUN_OFFSET_MASK         0x01ff0000
177 #define ZEND_MM_NRUN_OFFSET_OFFSET       16
178 
179 #define ZEND_MM_LRUN_PAGES(info)         (((info) & ZEND_MM_LRUN_PAGES_MASK) >> ZEND_MM_LRUN_PAGES_OFFSET)
180 #define ZEND_MM_SRUN_BIN_NUM(info)       (((info) & ZEND_MM_SRUN_BIN_NUM_MASK) >> ZEND_MM_SRUN_BIN_NUM_OFFSET)
181 #define ZEND_MM_SRUN_FREE_COUNTER(info)  (((info) & ZEND_MM_SRUN_FREE_COUNTER_MASK) >> ZEND_MM_SRUN_FREE_COUNTER_OFFSET)
182 #define ZEND_MM_NRUN_OFFSET(info)        (((info) & ZEND_MM_NRUN_OFFSET_MASK) >> ZEND_MM_NRUN_OFFSET_OFFSET)
183 
184 #define ZEND_MM_FRUN()                   ZEND_MM_IS_FRUN
185 #define ZEND_MM_LRUN(count)              (ZEND_MM_IS_LRUN | ((count) << ZEND_MM_LRUN_PAGES_OFFSET))
186 #define ZEND_MM_SRUN(bin_num)            (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET))
187 #define ZEND_MM_SRUN_EX(bin_num, count)  (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((count) << ZEND_MM_SRUN_FREE_COUNTER_OFFSET))
188 #define ZEND_MM_NRUN(bin_num, offset)    (ZEND_MM_IS_SRUN | ZEND_MM_IS_LRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((offset) << ZEND_MM_NRUN_OFFSET_OFFSET))
189 
190 #define ZEND_MM_BINS 30
191 
192 typedef struct  _zend_mm_page      zend_mm_page;
193 typedef struct  _zend_mm_bin       zend_mm_bin;
194 typedef struct  _zend_mm_free_slot zend_mm_free_slot;
195 typedef struct  _zend_mm_chunk     zend_mm_chunk;
196 typedef struct  _zend_mm_huge_list zend_mm_huge_list;
197 
198 int zend_mm_use_huge_pages = 0;
199 
200 /*
201  * Memory is retrieved from OS by chunks of fixed size 2MB.
202  * Inside chunk it's managed by pages of fixed size 4096B.
203  * So each chunk consists from 512 pages.
204  * The first page of each chunk is reserved for chunk header.
205  * It contains service information about all pages.
206  *
207  * free_pages - current number of free pages in this chunk
208  *
209  * free_tail  - number of continuous free pages at the end of chunk
210  *
211  * free_map   - bitset (a bit for each page). The bit is set if the corresponding
212  *              page is allocated. Allocator for "lage sizes" may easily find a
213  *              free page (or a continuous number of pages) searching for zero
214  *              bits.
215  *
216  * map        - contains service information for each page. (32-bits for each
217  *              page).
218  *    usage:
219  *				(2 bits)
220  * 				FRUN - free page,
221  *              LRUN - first page of "large" allocation
222  *              SRUN - first page of a bin used for "small" allocation
223  *
224  *    lrun_pages:
225  *              (10 bits) number of allocated pages
226  *
227  *    srun_bin_num:
228  *              (5 bits) bin number (e.g. 0 for sizes 0-2, 1 for 3-4,
229  *               2 for 5-8, 3 for 9-16 etc) see zend_alloc_sizes.h
230  */
231 
232 struct _zend_mm_heap {
233 #if ZEND_MM_CUSTOM
234 	int                use_custom_heap;
235 #endif
236 #if ZEND_MM_STORAGE
237 	zend_mm_storage   *storage;
238 #endif
239 #if ZEND_MM_STAT
240 	size_t             size;                    /* current memory usage */
241 	size_t             peak;                    /* peak memory usage */
242 #endif
243 	zend_mm_free_slot *free_slot[ZEND_MM_BINS]; /* free lists for small sizes */
244 #if ZEND_MM_STAT || ZEND_MM_LIMIT
245 	size_t             real_size;               /* current size of allocated pages */
246 #endif
247 #if ZEND_MM_STAT
248 	size_t             real_peak;               /* peak size of allocated pages */
249 #endif
250 #if ZEND_MM_LIMIT
251 	size_t             limit;                   /* memory limit */
252 	int                overflow;                /* memory overflow flag */
253 #endif
254 
255 	zend_mm_huge_list *huge_list;               /* list of huge allocated blocks */
256 
257 	zend_mm_chunk     *main_chunk;
258 	zend_mm_chunk     *cached_chunks;			/* list of unused chunks */
259 	int                chunks_count;			/* number of allocated chunks */
260 	int                peak_chunks_count;		/* peak number of allocated chunks for current request */
261 	int                cached_chunks_count;		/* number of cached chunks */
262 	double             avg_chunks_count;		/* average number of chunks allocated per request */
263 	int                last_chunks_delete_boundary; /* numer of chunks after last deletion */
264 	int                last_chunks_delete_count;    /* number of deletion over the last boundary */
265 #if ZEND_MM_CUSTOM
266 	union {
267 		struct {
268 			void      *(*_malloc)(size_t);
269 			void       (*_free)(void*);
270 			void      *(*_realloc)(void*, size_t);
271 		} std;
272 		struct {
273 			void      *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
274 			void       (*_free)(void*  ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
275 			void      *(*_realloc)(void*, size_t  ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
276 		} debug;
277 	} custom_heap;
278 	HashTable *tracked_allocs;
279 #endif
280 };
281 
282 struct _zend_mm_chunk {
283 	zend_mm_heap      *heap;
284 	zend_mm_chunk     *next;
285 	zend_mm_chunk     *prev;
286 	uint32_t           free_pages;				/* number of free pages */
287 	uint32_t           free_tail;               /* number of free pages at the end of chunk */
288 	uint32_t           num;
289 	char               reserve[64 - (sizeof(void*) * 3 + sizeof(uint32_t) * 3)];
290 	zend_mm_heap       heap_slot;               /* used only in main chunk */
291 	zend_mm_page_map   free_map;                /* 512 bits or 64 bytes */
292 	zend_mm_page_info  map[ZEND_MM_PAGES];      /* 2 KB = 512 * 4 */
293 };
294 
295 struct _zend_mm_page {
296 	char               bytes[ZEND_MM_PAGE_SIZE];
297 };
298 
299 /*
300  * bin - is one or few continuous pages (up to 8) used for allocation of
301  * a particular "small size".
302  */
303 struct _zend_mm_bin {
304 	char               bytes[ZEND_MM_PAGE_SIZE * 8];
305 };
306 
307 struct _zend_mm_free_slot {
308 	zend_mm_free_slot *next_free_slot;
309 };
310 
311 struct _zend_mm_huge_list {
312 	void              *ptr;
313 	size_t             size;
314 	zend_mm_huge_list *next;
315 #if ZEND_DEBUG
316 	zend_mm_debug_info dbg;
317 #endif
318 };
319 
320 #define ZEND_MM_PAGE_ADDR(chunk, page_num) \
321 	((void*)(((zend_mm_page*)(chunk)) + (page_num)))
322 
323 #define _BIN_DATA_SIZE(num, size, elements, pages, x, y) size,
324 static const uint32_t bin_data_size[] = {
325   ZEND_MM_BINS_INFO(_BIN_DATA_SIZE, x, y)
326 };
327 
328 #define _BIN_DATA_ELEMENTS(num, size, elements, pages, x, y) elements,
329 static const uint32_t bin_elements[] = {
330   ZEND_MM_BINS_INFO(_BIN_DATA_ELEMENTS, x, y)
331 };
332 
333 #define _BIN_DATA_PAGES(num, size, elements, pages, x, y) pages,
334 static const uint32_t bin_pages[] = {
335   ZEND_MM_BINS_INFO(_BIN_DATA_PAGES, x, y)
336 };
337 
338 #if ZEND_DEBUG
339 ZEND_COLD void zend_debug_alloc_output(char *format, ...)
340 {
341 	char output_buf[256];
342 	va_list args;
343 
344 	va_start(args, format);
345 	vsprintf(output_buf, format, args);
346 	va_end(args);
347 
348 #ifdef ZEND_WIN32
349 	OutputDebugString(output_buf);
350 #else
351 	fprintf(stderr, "%s", output_buf);
352 #endif
353 }
354 #endif
355 
356 static ZEND_COLD ZEND_NORETURN void zend_mm_panic(const char *message)
357 {
358 	fprintf(stderr, "%s\n", message);
359 /* See http://support.microsoft.com/kb/190351 */
360 #ifdef ZEND_WIN32
361 	fflush(stderr);
362 #endif
363 #if ZEND_DEBUG && defined(HAVE_KILL) && defined(HAVE_GETPID)
364 	kill(getpid(), SIGSEGV);
365 #endif
366 	exit(1);
367 }
368 
369 static ZEND_COLD ZEND_NORETURN void zend_mm_safe_error(zend_mm_heap *heap,
370 	const char *format,
371 	size_t limit,
372 #if ZEND_DEBUG
373 	const char *filename,
374 	uint32_t lineno,
375 #endif
376 	size_t size)
377 {
378 
379 	heap->overflow = 1;
380 	zend_try {
381 		zend_error_noreturn(E_ERROR,
382 			format,
383 			limit,
384 #if ZEND_DEBUG
385 			filename,
386 			lineno,
387 #endif
388 			size);
389 	} zend_catch {
390 	}  zend_end_try();
391 	heap->overflow = 0;
392 	zend_bailout();
393 	exit(1);
394 }
395 
396 #ifdef _WIN32
397 void
398 stderr_last_error(char *msg)
399 {
400 	DWORD err = GetLastError();
401 	char *buf = php_win32_error_to_msg(err);
402 
403 	if (!buf[0]) {
404 		fprintf(stderr, "\n%s: [0x%08lx]\n", msg, err);
405 	}
406 	else {
407 		fprintf(stderr, "\n%s: [0x%08lx] %s\n", msg, err, buf);
408 	}
409 
410 	php_win32_error_msg_free(buf);
411 }
412 #endif
413 
414 /*****************/
415 /* OS Allocation */
416 /*****************/
417 
418 #ifndef HAVE_MREMAP
419 static void *zend_mm_mmap_fixed(void *addr, size_t size)
420 {
421 #ifdef _WIN32
422 	return VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
423 #else
424 	int flags = MAP_PRIVATE | MAP_ANON;
425 #if defined(MAP_EXCL)
426 	flags |= MAP_FIXED | MAP_EXCL;
427 #endif
428 	/* MAP_FIXED leads to discarding of the old mapping, so it can't be used. */
429 	void *ptr = mmap(addr, size, PROT_READ | PROT_WRITE, flags /*| MAP_POPULATE | MAP_HUGETLB*/, ZEND_MM_FD, 0);
430 
431 	if (ptr == MAP_FAILED) {
432 #if ZEND_MM_ERROR && !defined(MAP_EXCL)
433 		fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno));
434 #endif
435 		return NULL;
436 	} else if (ptr != addr) {
437 		if (munmap(ptr, size) != 0) {
438 #if ZEND_MM_ERROR
439 			fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno));
440 #endif
441 		}
442 		return NULL;
443 	}
444 	return ptr;
445 #endif
446 }
447 #endif
448 
449 static void *zend_mm_mmap(size_t size)
450 {
451 #ifdef _WIN32
452 	void *ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
453 
454 	if (ptr == NULL) {
455 #if ZEND_MM_ERROR
456 		stderr_last_error("VirtualAlloc() failed");
457 #endif
458 		return NULL;
459 	}
460 	return ptr;
461 #else
462 	void *ptr;
463 
464 #ifdef MAP_HUGETLB
465 	if (zend_mm_use_huge_pages && size == ZEND_MM_CHUNK_SIZE) {
466 		ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_HUGETLB, -1, 0);
467 		if (ptr != MAP_FAILED) {
468 			return ptr;
469 		}
470 	}
471 #endif
472 
473 	ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, ZEND_MM_FD, 0);
474 
475 	if (ptr == MAP_FAILED) {
476 #if ZEND_MM_ERROR
477 		fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno));
478 #endif
479 		return NULL;
480 	}
481 	return ptr;
482 #endif
483 }
484 
485 static void zend_mm_munmap(void *addr, size_t size)
486 {
487 #ifdef _WIN32
488 	if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
489 #if ZEND_MM_ERROR
490 		stderr_last_error("VirtualFree() failed");
491 #endif
492 	}
493 #else
494 	if (munmap(addr, size) != 0) {
495 #if ZEND_MM_ERROR
496 		fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno));
497 #endif
498 	}
499 #endif
500 }
501 
502 /***********/
503 /* Bitmask */
504 /***********/
505 
506 /* number of trailing set (1) bits */
507 static zend_always_inline int zend_mm_bitset_nts(zend_mm_bitset bitset)
508 {
509 #if (defined(__GNUC__) || __has_builtin(__builtin_ctzl)) && SIZEOF_ZEND_LONG == SIZEOF_LONG && defined(PHP_HAVE_BUILTIN_CTZL)
510 	return __builtin_ctzl(~bitset);
511 #elif (defined(__GNUC__) || __has_builtin(__builtin_ctzll)) && defined(PHP_HAVE_BUILTIN_CTZLL)
512 	return __builtin_ctzll(~bitset);
513 #elif defined(_WIN32)
514 	unsigned long index;
515 
516 #if defined(_WIN64)
517 	if (!BitScanForward64(&index, ~bitset)) {
518 #else
519 	if (!BitScanForward(&index, ~bitset)) {
520 #endif
521 		/* undefined behavior */
522 		return 32;
523 	}
524 
525 	return (int)index;
526 #else
527 	int n;
528 
529 	if (bitset == (zend_mm_bitset)-1) return ZEND_MM_BITSET_LEN;
530 
531 	n = 0;
532 #if SIZEOF_ZEND_LONG == 8
533 	if (sizeof(zend_mm_bitset) == 8) {
534 		if ((bitset & 0xffffffff) == 0xffffffff) {n += 32; bitset = bitset >> Z_UL(32);}
535 	}
536 #endif
537 	if ((bitset & 0x0000ffff) == 0x0000ffff) {n += 16; bitset = bitset >> 16;}
538 	if ((bitset & 0x000000ff) == 0x000000ff) {n +=  8; bitset = bitset >>  8;}
539 	if ((bitset & 0x0000000f) == 0x0000000f) {n +=  4; bitset = bitset >>  4;}
540 	if ((bitset & 0x00000003) == 0x00000003) {n +=  2; bitset = bitset >>  2;}
541 	return n + (bitset & 1);
542 #endif
543 }
544 
545 static zend_always_inline int zend_mm_bitset_is_set(zend_mm_bitset *bitset, int bit)
546 {
547 	return ZEND_BIT_TEST(bitset, bit);
548 }
549 
550 static zend_always_inline void zend_mm_bitset_set_bit(zend_mm_bitset *bitset, int bit)
551 {
552 	bitset[bit / ZEND_MM_BITSET_LEN] |= (Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
553 }
554 
555 static zend_always_inline void zend_mm_bitset_reset_bit(zend_mm_bitset *bitset, int bit)
556 {
557 	bitset[bit / ZEND_MM_BITSET_LEN] &= ~(Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
558 }
559 
560 static zend_always_inline void zend_mm_bitset_set_range(zend_mm_bitset *bitset, int start, int len)
561 {
562 	if (len == 1) {
563 		zend_mm_bitset_set_bit(bitset, start);
564 	} else {
565 		int pos = start / ZEND_MM_BITSET_LEN;
566 		int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
567 		int bit = start & (ZEND_MM_BITSET_LEN - 1);
568 		zend_mm_bitset tmp;
569 
570 		if (pos != end) {
571 			/* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
572 			tmp = (zend_mm_bitset)-1 << bit;
573 			bitset[pos++] |= tmp;
574 			while (pos != end) {
575 				/* set all bits */
576 				bitset[pos++] = (zend_mm_bitset)-1;
577 			}
578 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
579 			/* set bits from "0" to "end" */
580 			tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
581 			bitset[pos] |= tmp;
582 		} else {
583 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
584 			/* set bits from "bit" to "end" */
585 			tmp = (zend_mm_bitset)-1 << bit;
586 			tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
587 			bitset[pos] |= tmp;
588 		}
589 	}
590 }
591 
592 static zend_always_inline void zend_mm_bitset_reset_range(zend_mm_bitset *bitset, int start, int len)
593 {
594 	if (len == 1) {
595 		zend_mm_bitset_reset_bit(bitset, start);
596 	} else {
597 		int pos = start / ZEND_MM_BITSET_LEN;
598 		int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
599 		int bit = start & (ZEND_MM_BITSET_LEN - 1);
600 		zend_mm_bitset tmp;
601 
602 		if (pos != end) {
603 			/* reset bits from "bit" to ZEND_MM_BITSET_LEN-1 */
604 			tmp = ~((Z_UL(1) << bit) - 1);
605 			bitset[pos++] &= ~tmp;
606 			while (pos != end) {
607 				/* set all bits */
608 				bitset[pos++] = 0;
609 			}
610 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
611 			/* reset bits from "0" to "end" */
612 			tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
613 			bitset[pos] &= ~tmp;
614 		} else {
615 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
616 			/* reset bits from "bit" to "end" */
617 			tmp = (zend_mm_bitset)-1 << bit;
618 			tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
619 			bitset[pos] &= ~tmp;
620 		}
621 	}
622 }
623 
624 static zend_always_inline int zend_mm_bitset_is_free_range(zend_mm_bitset *bitset, int start, int len)
625 {
626 	if (len == 1) {
627 		return !zend_mm_bitset_is_set(bitset, start);
628 	} else {
629 		int pos = start / ZEND_MM_BITSET_LEN;
630 		int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
631 		int bit = start & (ZEND_MM_BITSET_LEN - 1);
632 		zend_mm_bitset tmp;
633 
634 		if (pos != end) {
635 			/* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
636 			tmp = (zend_mm_bitset)-1 << bit;
637 			if ((bitset[pos++] & tmp) != 0) {
638 				return 0;
639 			}
640 			while (pos != end) {
641 				/* set all bits */
642 				if (bitset[pos++] != 0) {
643 					return 0;
644 				}
645 			}
646 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
647 			/* set bits from "0" to "end" */
648 			tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
649 			return (bitset[pos] & tmp) == 0;
650 		} else {
651 			end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
652 			/* set bits from "bit" to "end" */
653 			tmp = (zend_mm_bitset)-1 << bit;
654 			tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
655 			return (bitset[pos] & tmp) == 0;
656 		}
657 	}
658 }
659 
660 /**********/
661 /* Chunks */
662 /**********/
663 
664 static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment)
665 {
666 	void *ptr = zend_mm_mmap(size);
667 
668 	if (ptr == NULL) {
669 		return NULL;
670 	} else if (ZEND_MM_ALIGNED_OFFSET(ptr, alignment) == 0) {
671 #ifdef MADV_HUGEPAGE
672 		if (zend_mm_use_huge_pages) {
673 			madvise(ptr, size, MADV_HUGEPAGE);
674 		}
675 #endif
676 		return ptr;
677 	} else {
678 		size_t offset;
679 
680 		/* chunk has to be aligned */
681 		zend_mm_munmap(ptr, size);
682 		ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
683 #ifdef _WIN32
684 		offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
685 		zend_mm_munmap(ptr, size + alignment - REAL_PAGE_SIZE);
686 		ptr = zend_mm_mmap_fixed((void*)((char*)ptr + (alignment - offset)), size);
687 		offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
688 		if (offset != 0) {
689 			zend_mm_munmap(ptr, size);
690 			return NULL;
691 		}
692 		return ptr;
693 #else
694 		offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
695 		if (offset != 0) {
696 			offset = alignment - offset;
697 			zend_mm_munmap(ptr, offset);
698 			ptr = (char*)ptr + offset;
699 			alignment -= offset;
700 		}
701 		if (alignment > REAL_PAGE_SIZE) {
702 			zend_mm_munmap((char*)ptr + size, alignment - REAL_PAGE_SIZE);
703 		}
704 # ifdef MADV_HUGEPAGE
705 		if (zend_mm_use_huge_pages) {
706 			madvise(ptr, size, MADV_HUGEPAGE);
707 		}
708 # endif
709 #endif
710 		return ptr;
711 	}
712 }
713 
714 static void *zend_mm_chunk_alloc(zend_mm_heap *heap, size_t size, size_t alignment)
715 {
716 #if ZEND_MM_STORAGE
717 	if (UNEXPECTED(heap->storage)) {
718 		void *ptr = heap->storage->handlers.chunk_alloc(heap->storage, size, alignment);
719 		ZEND_ASSERT(((zend_uintptr_t)((char*)ptr + (alignment-1)) & (alignment-1)) == (zend_uintptr_t)ptr);
720 		return ptr;
721 	}
722 #endif
723 	return zend_mm_chunk_alloc_int(size, alignment);
724 }
725 
726 static void zend_mm_chunk_free(zend_mm_heap *heap, void *addr, size_t size)
727 {
728 #if ZEND_MM_STORAGE
729 	if (UNEXPECTED(heap->storage)) {
730 		heap->storage->handlers.chunk_free(heap->storage, addr, size);
731 		return;
732 	}
733 #endif
734 	zend_mm_munmap(addr, size);
735 }
736 
737 static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
738 {
739 #if ZEND_MM_STORAGE
740 	if (UNEXPECTED(heap->storage)) {
741 		if (heap->storage->handlers.chunk_truncate) {
742 			return heap->storage->handlers.chunk_truncate(heap->storage, addr, old_size, new_size);
743 		} else {
744 			return 0;
745 		}
746 	}
747 #endif
748 #ifndef _WIN32
749 	zend_mm_munmap((char*)addr + new_size, old_size - new_size);
750 	return 1;
751 #else
752 	return 0;
753 #endif
754 }
755 
756 static int zend_mm_chunk_extend(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
757 {
758 #if ZEND_MM_STORAGE
759 	if (UNEXPECTED(heap->storage)) {
760 		if (heap->storage->handlers.chunk_extend) {
761 			return heap->storage->handlers.chunk_extend(heap->storage, addr, old_size, new_size);
762 		} else {
763 			return 0;
764 		}
765 	}
766 #endif
767 #ifdef HAVE_MREMAP
768 	/* We don't use MREMAP_MAYMOVE due to alignment requirements. */
769 	void *ptr = mremap(addr, old_size, new_size, 0);
770 	if (ptr == MAP_FAILED) {
771 		return 0;
772 	}
773 	/* Sanity check: The mapping shouldn't have moved. */
774 	ZEND_ASSERT(ptr == addr);
775 	return 1;
776 #elif !defined(_WIN32)
777 	return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL);
778 #else
779 	return 0;
780 #endif
781 }
782 
783 static zend_always_inline void zend_mm_chunk_init(zend_mm_heap *heap, zend_mm_chunk *chunk)
784 {
785 	chunk->heap = heap;
786 	chunk->next = heap->main_chunk;
787 	chunk->prev = heap->main_chunk->prev;
788 	chunk->prev->next = chunk;
789 	chunk->next->prev = chunk;
790 	/* mark first pages as allocated */
791 	chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
792 	chunk->free_tail = ZEND_MM_FIRST_PAGE;
793 	/* the younger chunks have bigger number */
794 	chunk->num = chunk->prev->num + 1;
795 	/* mark first pages as allocated */
796 	chunk->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
797 	chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
798 }
799 
800 /***********************/
801 /* Huge Runs (forward) */
802 /***********************/
803 
804 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
805 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
806 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
807 
808 #if ZEND_DEBUG
809 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
810 #else
811 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
812 #endif
813 
814 /**************/
815 /* Large Runs */
816 /**************/
817 
818 #if ZEND_DEBUG
819 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
820 #else
821 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
822 #endif
823 {
824 	zend_mm_chunk *chunk = heap->main_chunk;
825 	uint32_t page_num, len;
826 	int steps = 0;
827 
828 	while (1) {
829 		if (UNEXPECTED(chunk->free_pages < pages_count)) {
830 			goto not_found;
831 #if 0
832 		} else if (UNEXPECTED(chunk->free_pages + chunk->free_tail == ZEND_MM_PAGES)) {
833 			if (UNEXPECTED(ZEND_MM_PAGES - chunk->free_tail < pages_count)) {
834 				goto not_found;
835 			} else {
836 				page_num = chunk->free_tail;
837 				goto found;
838 			}
839 		} else if (0) {
840 			/* First-Fit Search */
841 			int free_tail = chunk->free_tail;
842 			zend_mm_bitset *bitset = chunk->free_map;
843 			zend_mm_bitset tmp = *(bitset++);
844 			int i = 0;
845 
846 			while (1) {
847 				/* skip allocated blocks */
848 				while (tmp == (zend_mm_bitset)-1) {
849 					i += ZEND_MM_BITSET_LEN;
850 					if (i == ZEND_MM_PAGES) {
851 						goto not_found;
852 					}
853 					tmp = *(bitset++);
854 				}
855 				/* find first 0 bit */
856 				page_num = i + zend_mm_bitset_nts(tmp);
857 				/* reset bits from 0 to "bit" */
858 				tmp &= tmp + 1;
859 				/* skip free blocks */
860 				while (tmp == 0) {
861 					i += ZEND_MM_BITSET_LEN;
862 					len = i - page_num;
863 					if (len >= pages_count) {
864 						goto found;
865 					} else if (i >= free_tail) {
866 						goto not_found;
867 					}
868 					tmp = *(bitset++);
869 				}
870 				/* find first 1 bit */
871 				len = (i + zend_ulong_ntz(tmp)) - page_num;
872 				if (len >= pages_count) {
873 					goto found;
874 				}
875 				/* set bits from 0 to "bit" */
876 				tmp |= tmp - 1;
877 			}
878 #endif
879 		} else {
880 			/* Best-Fit Search */
881 			int best = -1;
882 			uint32_t best_len = ZEND_MM_PAGES;
883 			uint32_t free_tail = chunk->free_tail;
884 			zend_mm_bitset *bitset = chunk->free_map;
885 			zend_mm_bitset tmp = *(bitset++);
886 			uint32_t i = 0;
887 
888 			while (1) {
889 				/* skip allocated blocks */
890 				while (tmp == (zend_mm_bitset)-1) {
891 					i += ZEND_MM_BITSET_LEN;
892 					if (i == ZEND_MM_PAGES) {
893 						if (best > 0) {
894 							page_num = best;
895 							goto found;
896 						} else {
897 							goto not_found;
898 						}
899 					}
900 					tmp = *(bitset++);
901 				}
902 				/* find first 0 bit */
903 				page_num = i + zend_mm_bitset_nts(tmp);
904 				/* reset bits from 0 to "bit" */
905 				tmp &= tmp + 1;
906 				/* skip free blocks */
907 				while (tmp == 0) {
908 					i += ZEND_MM_BITSET_LEN;
909 					if (i >= free_tail || i == ZEND_MM_PAGES) {
910 						len = ZEND_MM_PAGES - page_num;
911 						if (len >= pages_count && len < best_len) {
912 							chunk->free_tail = page_num + pages_count;
913 							goto found;
914 						} else {
915 							/* set accurate value */
916 							chunk->free_tail = page_num;
917 							if (best > 0) {
918 								page_num = best;
919 								goto found;
920 							} else {
921 								goto not_found;
922 							}
923 						}
924 					}
925 					tmp = *(bitset++);
926 				}
927 				/* find first 1 bit */
928 				len = i + zend_ulong_ntz(tmp) - page_num;
929 				if (len >= pages_count) {
930 					if (len == pages_count) {
931 						goto found;
932 					} else if (len < best_len) {
933 						best_len = len;
934 						best = page_num;
935 					}
936 				}
937 				/* set bits from 0 to "bit" */
938 				tmp |= tmp - 1;
939 			}
940 		}
941 
942 not_found:
943 		if (chunk->next == heap->main_chunk) {
944 get_chunk:
945 			if (heap->cached_chunks) {
946 				heap->cached_chunks_count--;
947 				chunk = heap->cached_chunks;
948 				heap->cached_chunks = chunk->next;
949 			} else {
950 #if ZEND_MM_LIMIT
951 				if (UNEXPECTED(ZEND_MM_CHUNK_SIZE > heap->limit - heap->real_size)) {
952 					if (zend_mm_gc(heap)) {
953 						goto get_chunk;
954 					} else if (heap->overflow == 0) {
955 #if ZEND_DEBUG
956 						zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
957 #else
958 						zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, ZEND_MM_PAGE_SIZE * pages_count);
959 #endif
960 						return NULL;
961 					}
962 				}
963 #endif
964 				chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
965 				if (UNEXPECTED(chunk == NULL)) {
966 					/* insufficient memory */
967 					if (zend_mm_gc(heap) &&
968 					    (chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE)) != NULL) {
969 						/* pass */
970 					} else {
971 #if !ZEND_MM_LIMIT
972 						zend_mm_safe_error(heap, "Out of memory");
973 #elif ZEND_DEBUG
974 						zend_mm_safe_error(heap, "Out of memory (allocated %zu) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
975 #else
976 						zend_mm_safe_error(heap, "Out of memory (allocated %zu) (tried to allocate %zu bytes)", heap->real_size, ZEND_MM_PAGE_SIZE * pages_count);
977 #endif
978 						return NULL;
979 					}
980 				}
981 #if ZEND_MM_STAT
982 				do {
983 					size_t size = heap->real_size + ZEND_MM_CHUNK_SIZE;
984 					size_t peak = MAX(heap->real_peak, size);
985 					heap->real_size = size;
986 					heap->real_peak = peak;
987 				} while (0);
988 #elif ZEND_MM_LIMIT
989 				heap->real_size += ZEND_MM_CHUNK_SIZE;
990 
991 #endif
992 			}
993 			heap->chunks_count++;
994 			if (heap->chunks_count > heap->peak_chunks_count) {
995 				heap->peak_chunks_count = heap->chunks_count;
996 			}
997 			zend_mm_chunk_init(heap, chunk);
998 			page_num = ZEND_MM_FIRST_PAGE;
999 			len = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1000 			goto found;
1001 		} else {
1002 			chunk = chunk->next;
1003 			steps++;
1004 		}
1005 	}
1006 
1007 found:
1008 	if (steps > 2 && pages_count < 8) {
1009 		/* move chunk into the head of the linked-list */
1010 		chunk->prev->next = chunk->next;
1011 		chunk->next->prev = chunk->prev;
1012 		chunk->next = heap->main_chunk->next;
1013 		chunk->prev = heap->main_chunk;
1014 		chunk->prev->next = chunk;
1015 		chunk->next->prev = chunk;
1016 	}
1017 	/* mark run as allocated */
1018 	chunk->free_pages -= pages_count;
1019 	zend_mm_bitset_set_range(chunk->free_map, page_num, pages_count);
1020 	chunk->map[page_num] = ZEND_MM_LRUN(pages_count);
1021 	if (page_num == chunk->free_tail) {
1022 		chunk->free_tail = page_num + pages_count;
1023 	}
1024 	return ZEND_MM_PAGE_ADDR(chunk, page_num);
1025 }
1026 
1027 static zend_always_inline void *zend_mm_alloc_large_ex(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1028 {
1029 	int pages_count = (int)ZEND_MM_SIZE_TO_NUM(size, ZEND_MM_PAGE_SIZE);
1030 #if ZEND_DEBUG
1031 	void *ptr = zend_mm_alloc_pages(heap, pages_count, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1032 #else
1033 	void *ptr = zend_mm_alloc_pages(heap, pages_count ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1034 #endif
1035 #if ZEND_MM_STAT
1036 	do {
1037 		size_t size = heap->size + pages_count * ZEND_MM_PAGE_SIZE;
1038 		size_t peak = MAX(heap->peak, size);
1039 		heap->size = size;
1040 		heap->peak = peak;
1041 	} while (0);
1042 #endif
1043 	return ptr;
1044 }
1045 
1046 #if ZEND_DEBUG
1047 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1048 {
1049 	return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1050 }
1051 #else
1052 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1053 {
1054 	return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1055 }
1056 #endif
1057 
1058 static zend_always_inline void zend_mm_delete_chunk(zend_mm_heap *heap, zend_mm_chunk *chunk)
1059 {
1060 	chunk->next->prev = chunk->prev;
1061 	chunk->prev->next = chunk->next;
1062 	heap->chunks_count--;
1063 	if (heap->chunks_count + heap->cached_chunks_count < heap->avg_chunks_count + 0.1
1064 	 || (heap->chunks_count == heap->last_chunks_delete_boundary
1065 	  && heap->last_chunks_delete_count >= 4)) {
1066 		/* delay deletion */
1067 		heap->cached_chunks_count++;
1068 		chunk->next = heap->cached_chunks;
1069 		heap->cached_chunks = chunk;
1070 	} else {
1071 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1072 		heap->real_size -= ZEND_MM_CHUNK_SIZE;
1073 #endif
1074 		if (!heap->cached_chunks) {
1075 			if (heap->chunks_count != heap->last_chunks_delete_boundary) {
1076 				heap->last_chunks_delete_boundary = heap->chunks_count;
1077 				heap->last_chunks_delete_count = 0;
1078 			} else {
1079 				heap->last_chunks_delete_count++;
1080 			}
1081 		}
1082 		if (!heap->cached_chunks || chunk->num > heap->cached_chunks->num) {
1083 			zend_mm_chunk_free(heap, chunk, ZEND_MM_CHUNK_SIZE);
1084 		} else {
1085 //TODO: select the best chunk to delete???
1086 			chunk->next = heap->cached_chunks->next;
1087 			zend_mm_chunk_free(heap, heap->cached_chunks, ZEND_MM_CHUNK_SIZE);
1088 			heap->cached_chunks = chunk;
1089 		}
1090 	}
1091 }
1092 
1093 static zend_always_inline void zend_mm_free_pages_ex(zend_mm_heap *heap, zend_mm_chunk *chunk, uint32_t page_num, uint32_t pages_count, int free_chunk)
1094 {
1095 	chunk->free_pages += pages_count;
1096 	zend_mm_bitset_reset_range(chunk->free_map, page_num, pages_count);
1097 	chunk->map[page_num] = 0;
1098 	if (chunk->free_tail == page_num + pages_count) {
1099 		/* this setting may be not accurate */
1100 		chunk->free_tail = page_num;
1101 	}
1102 	if (free_chunk && chunk != heap->main_chunk && chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
1103 		zend_mm_delete_chunk(heap, chunk);
1104 	}
1105 }
1106 
1107 static zend_never_inline void zend_mm_free_pages(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1108 {
1109 	zend_mm_free_pages_ex(heap, chunk, page_num, pages_count, 1);
1110 }
1111 
1112 static zend_always_inline void zend_mm_free_large(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1113 {
1114 #if ZEND_MM_STAT
1115 	heap->size -= pages_count * ZEND_MM_PAGE_SIZE;
1116 #endif
1117 	zend_mm_free_pages(heap, chunk, page_num, pages_count);
1118 }
1119 
1120 /**************/
1121 /* Small Runs */
1122 /**************/
1123 
1124 /* higher set bit number (0->N/A, 1->1, 2->2, 4->3, 8->4, 127->7, 128->8 etc) */
1125 static zend_always_inline int zend_mm_small_size_to_bit(int size)
1126 {
1127 #if (defined(__GNUC__) || __has_builtin(__builtin_clz))  && defined(PHP_HAVE_BUILTIN_CLZ)
1128 	return (__builtin_clz(size) ^ 0x1f) + 1;
1129 #elif defined(_WIN32)
1130 	unsigned long index;
1131 
1132 	if (!BitScanReverse(&index, (unsigned long)size)) {
1133 		/* undefined behavior */
1134 		return 64;
1135 	}
1136 
1137 	return (((31 - (int)index) ^ 0x1f) + 1);
1138 #else
1139 	int n = 16;
1140 	if (size <= 0x00ff) {n -= 8; size = size << 8;}
1141 	if (size <= 0x0fff) {n -= 4; size = size << 4;}
1142 	if (size <= 0x3fff) {n -= 2; size = size << 2;}
1143 	if (size <= 0x7fff) {n -= 1;}
1144 	return n;
1145 #endif
1146 }
1147 
1148 #ifndef MAX
1149 # define MAX(a, b) (((a) > (b)) ? (a) : (b))
1150 #endif
1151 
1152 #ifndef MIN
1153 # define MIN(a, b) (((a) < (b)) ? (a) : (b))
1154 #endif
1155 
1156 static zend_always_inline int zend_mm_small_size_to_bin(size_t size)
1157 {
1158 #if 0
1159 	int n;
1160                             /*0,  1,  2,  3,  4,  5,  6,  7,  8,  9  10, 11, 12*/
1161 	static const int f1[] = { 3,  3,  3,  3,  3,  3,  3,  4,  5,  6,  7,  8,  9};
1162 	static const int f2[] = { 0,  0,  0,  0,  0,  0,  0,  4,  8, 12, 16, 20, 24};
1163 
1164 	if (UNEXPECTED(size <= 2)) return 0;
1165 	n = zend_mm_small_size_to_bit(size - 1);
1166 	return ((size-1) >> f1[n]) + f2[n];
1167 #else
1168 	unsigned int t1, t2;
1169 
1170 	if (size <= 64) {
1171 		/* we need to support size == 0 ... */
1172 		return (size - !!size) >> 3;
1173 	} else {
1174 		t1 = size - 1;
1175 		t2 = zend_mm_small_size_to_bit(t1) - 3;
1176 		t1 = t1 >> t2;
1177 		t2 = t2 - 3;
1178 		t2 = t2 << 2;
1179 		return (int)(t1 + t2);
1180 	}
1181 #endif
1182 }
1183 
1184 #define ZEND_MM_SMALL_SIZE_TO_BIN(size)  zend_mm_small_size_to_bin(size)
1185 
1186 static zend_never_inline void *zend_mm_alloc_small_slow(zend_mm_heap *heap, uint32_t bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1187 {
1188     zend_mm_chunk *chunk;
1189     int page_num;
1190 	zend_mm_bin *bin;
1191 	zend_mm_free_slot *p, *end;
1192 
1193 #if ZEND_DEBUG
1194 	bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num], bin_data_size[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1195 #else
1196 	bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1197 #endif
1198 	if (UNEXPECTED(bin == NULL)) {
1199 		/* insufficient memory */
1200 		return NULL;
1201 	}
1202 
1203 	chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(bin, ZEND_MM_CHUNK_SIZE);
1204 	page_num = ZEND_MM_ALIGNED_OFFSET(bin, ZEND_MM_CHUNK_SIZE) / ZEND_MM_PAGE_SIZE;
1205 	chunk->map[page_num] = ZEND_MM_SRUN(bin_num);
1206 	if (bin_pages[bin_num] > 1) {
1207 		uint32_t i = 1;
1208 
1209 		do {
1210 			chunk->map[page_num+i] = ZEND_MM_NRUN(bin_num, i);
1211 			i++;
1212 		} while (i < bin_pages[bin_num]);
1213 	}
1214 
1215 	/* create a linked list of elements from 1 to last */
1216 	end = (zend_mm_free_slot*)((char*)bin + (bin_data_size[bin_num] * (bin_elements[bin_num] - 1)));
1217 	heap->free_slot[bin_num] = p = (zend_mm_free_slot*)((char*)bin + bin_data_size[bin_num]);
1218 	do {
1219 		p->next_free_slot = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1220 #if ZEND_DEBUG
1221 		do {
1222 			zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1223 			dbg->size = 0;
1224 		} while (0);
1225 #endif
1226 		p = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1227 	} while (p != end);
1228 
1229 	/* terminate list using NULL */
1230 	p->next_free_slot = NULL;
1231 #if ZEND_DEBUG
1232 		do {
1233 			zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1234 			dbg->size = 0;
1235 		} while (0);
1236 #endif
1237 
1238 	/* return first element */
1239 	return (char*)bin;
1240 }
1241 
1242 static zend_always_inline void *zend_mm_alloc_small(zend_mm_heap *heap, int bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1243 {
1244 #if ZEND_MM_STAT
1245 	do {
1246 		size_t size = heap->size + bin_data_size[bin_num];
1247 		size_t peak = MAX(heap->peak, size);
1248 		heap->size = size;
1249 		heap->peak = peak;
1250 	} while (0);
1251 #endif
1252 
1253 	if (EXPECTED(heap->free_slot[bin_num] != NULL)) {
1254 		zend_mm_free_slot *p = heap->free_slot[bin_num];
1255 		heap->free_slot[bin_num] = p->next_free_slot;
1256 		return (void*)p;
1257 	} else {
1258 		return zend_mm_alloc_small_slow(heap, bin_num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1259 	}
1260 }
1261 
1262 static zend_always_inline void zend_mm_free_small(zend_mm_heap *heap, void *ptr, int bin_num)
1263 {
1264 	zend_mm_free_slot *p;
1265 
1266 #if ZEND_MM_STAT
1267 	heap->size -= bin_data_size[bin_num];
1268 #endif
1269 
1270 #if ZEND_DEBUG
1271 	do {
1272 		zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1273 		dbg->size = 0;
1274 	} while (0);
1275 #endif
1276 
1277     p = (zend_mm_free_slot*)ptr;
1278     p->next_free_slot = heap->free_slot[bin_num];
1279     heap->free_slot[bin_num] = p;
1280 }
1281 
1282 /********/
1283 /* Heap */
1284 /********/
1285 
1286 #if ZEND_DEBUG
1287 static zend_always_inline zend_mm_debug_info *zend_mm_get_debug_info(zend_mm_heap *heap, void *ptr)
1288 {
1289 	size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1290 	zend_mm_chunk *chunk;
1291 	int page_num;
1292 	zend_mm_page_info info;
1293 
1294 	ZEND_MM_CHECK(page_offset != 0, "zend_mm_heap corrupted");
1295 	chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1296 	page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1297 	info = chunk->map[page_num];
1298 	ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1299 	if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1300 		int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1301 		return (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1302 	} else /* if (info & ZEND_MM_IS_LRUN) */ {
1303 		int pages_count = ZEND_MM_LRUN_PAGES(info);
1304 
1305 		return (zend_mm_debug_info*)((char*)ptr + ZEND_MM_PAGE_SIZE * pages_count - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1306 	}
1307 }
1308 #endif
1309 
1310 static zend_always_inline void *zend_mm_alloc_heap(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1311 {
1312 	void *ptr;
1313 #if ZEND_DEBUG
1314 	size_t real_size = size;
1315 	zend_mm_debug_info *dbg;
1316 
1317 	/* special handling for zero-size allocation */
1318 	size = MAX(size, 1);
1319 	size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1320 	if (UNEXPECTED(size < real_size)) {
1321 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", ZEND_MM_ALIGNED_SIZE(real_size), ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1322 		return NULL;
1323 	}
1324 #endif
1325 	if (EXPECTED(size <= ZEND_MM_MAX_SMALL_SIZE)) {
1326 		ptr = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1327 #if ZEND_DEBUG
1328 		dbg = zend_mm_get_debug_info(heap, ptr);
1329 		dbg->size = real_size;
1330 		dbg->filename = __zend_filename;
1331 		dbg->orig_filename = __zend_orig_filename;
1332 		dbg->lineno = __zend_lineno;
1333 		dbg->orig_lineno = __zend_orig_lineno;
1334 #endif
1335 		return ptr;
1336 	} else if (EXPECTED(size <= ZEND_MM_MAX_LARGE_SIZE)) {
1337 		ptr = zend_mm_alloc_large(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1338 #if ZEND_DEBUG
1339 		dbg = zend_mm_get_debug_info(heap, ptr);
1340 		dbg->size = real_size;
1341 		dbg->filename = __zend_filename;
1342 		dbg->orig_filename = __zend_orig_filename;
1343 		dbg->lineno = __zend_lineno;
1344 		dbg->orig_lineno = __zend_orig_lineno;
1345 #endif
1346 		return ptr;
1347 	} else {
1348 #if ZEND_DEBUG
1349 		size = real_size;
1350 #endif
1351 		return zend_mm_alloc_huge(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1352 	}
1353 }
1354 
1355 static zend_always_inline void zend_mm_free_heap(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1356 {
1357 	size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1358 
1359 	if (UNEXPECTED(page_offset == 0)) {
1360 		if (ptr != NULL) {
1361 			zend_mm_free_huge(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1362 		}
1363 	} else {
1364 		zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1365 		int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1366 		zend_mm_page_info info = chunk->map[page_num];
1367 
1368 		ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1369 		if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1370 			zend_mm_free_small(heap, ptr, ZEND_MM_SRUN_BIN_NUM(info));
1371 		} else /* if (info & ZEND_MM_IS_LRUN) */ {
1372 			int pages_count = ZEND_MM_LRUN_PAGES(info);
1373 
1374 			ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1375 			zend_mm_free_large(heap, chunk, page_num, pages_count);
1376 		}
1377 	}
1378 }
1379 
1380 static size_t zend_mm_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1381 {
1382 	size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1383 
1384 	if (UNEXPECTED(page_offset == 0)) {
1385 		return zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1386 	} else {
1387 		zend_mm_chunk *chunk;
1388 #if 0 && ZEND_DEBUG
1389 		zend_mm_debug_info *dbg = zend_mm_get_debug_info(heap, ptr);
1390 		return dbg->size;
1391 #else
1392 		int page_num;
1393 		zend_mm_page_info info;
1394 
1395 		chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1396 		page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1397 		info = chunk->map[page_num];
1398 		ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1399 		if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1400 			return bin_data_size[ZEND_MM_SRUN_BIN_NUM(info)];
1401 		} else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1402 			return ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1403 		}
1404 #endif
1405 	}
1406 }
1407 
1408 static zend_never_inline void *zend_mm_realloc_slow(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1409 {
1410 	void *ret;
1411 
1412 #if ZEND_MM_STAT
1413 	do {
1414 		size_t orig_peak = heap->peak;
1415 #endif
1416 		ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1417 		memcpy(ret, ptr, copy_size);
1418 		zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1419 #if ZEND_MM_STAT
1420 		heap->peak = MAX(orig_peak, heap->size);
1421 	} while (0);
1422 #endif
1423 	return ret;
1424 }
1425 
1426 static zend_never_inline void *zend_mm_realloc_huge(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1427 {
1428 	size_t old_size;
1429 	size_t new_size;
1430 #if ZEND_DEBUG
1431 	size_t real_size;
1432 #endif
1433 
1434 	old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1435 #if ZEND_DEBUG
1436 	real_size = size;
1437 	size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1438 #endif
1439 	if (size > ZEND_MM_MAX_LARGE_SIZE) {
1440 #if ZEND_DEBUG
1441 		size = real_size;
1442 #endif
1443 #ifdef ZEND_WIN32
1444 		/* On Windows we don't have ability to extend huge blocks in-place.
1445 		 * We allocate them with 2MB size granularity, to avoid many
1446 		 * reallocations when they are extended by small pieces
1447 		 */
1448 		new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE));
1449 #else
1450 		new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE);
1451 #endif
1452 		if (new_size == old_size) {
1453 #if ZEND_DEBUG
1454 			zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1455 #else
1456 			zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1457 #endif
1458 			return ptr;
1459 		} else if (new_size < old_size) {
1460 			/* unmup tail */
1461 			if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) {
1462 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1463 				heap->real_size -= old_size - new_size;
1464 #endif
1465 #if ZEND_MM_STAT
1466 				heap->size -= old_size - new_size;
1467 #endif
1468 #if ZEND_DEBUG
1469 				zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1470 #else
1471 				zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1472 #endif
1473 				return ptr;
1474 			}
1475 		} else /* if (new_size > old_size) */ {
1476 #if ZEND_MM_LIMIT
1477 			if (UNEXPECTED(new_size - old_size > heap->limit - heap->real_size)) {
1478 				if (zend_mm_gc(heap) && new_size - old_size <= heap->limit - heap->real_size) {
1479 					/* pass */
1480 				} else if (heap->overflow == 0) {
1481 #if ZEND_DEBUG
1482 					zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1483 #else
1484 					zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1485 #endif
1486 					return NULL;
1487 				}
1488 			}
1489 #endif
1490 			/* try to map tail right after this block */
1491 			if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) {
1492 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1493 				heap->real_size += new_size - old_size;
1494 #endif
1495 #if ZEND_MM_STAT
1496 				heap->real_peak = MAX(heap->real_peak, heap->real_size);
1497 				heap->size += new_size - old_size;
1498 				heap->peak = MAX(heap->peak, heap->size);
1499 #endif
1500 #if ZEND_DEBUG
1501 				zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1502 #else
1503 				zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1504 #endif
1505 				return ptr;
1506 			}
1507 		}
1508 	}
1509 
1510 	return zend_mm_realloc_slow(heap, ptr, size, MIN(old_size, copy_size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1511 }
1512 
1513 static zend_always_inline void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, zend_bool use_copy_size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1514 {
1515 	size_t page_offset;
1516 	size_t old_size;
1517 	size_t new_size;
1518 	void *ret;
1519 #if ZEND_DEBUG
1520 	zend_mm_debug_info *dbg;
1521 #endif
1522 
1523 	page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1524 	if (UNEXPECTED(page_offset == 0)) {
1525 		if (EXPECTED(ptr == NULL)) {
1526 			return _zend_mm_alloc(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1527 		} else {
1528 			return zend_mm_realloc_huge(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1529 		}
1530 	} else {
1531 		zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1532 		int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1533 		zend_mm_page_info info = chunk->map[page_num];
1534 #if ZEND_DEBUG
1535 		size_t real_size = size;
1536 
1537 		size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1538 #endif
1539 
1540 		ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1541 		if (info & ZEND_MM_IS_SRUN) {
1542 			int old_bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1543 
1544 			do {
1545 				old_size = bin_data_size[old_bin_num];
1546 
1547 				/* Check if requested size fits into current bin */
1548 				if (size <= old_size) {
1549 					/* Check if truncation is necessary */
1550 					if (old_bin_num > 0 && size < bin_data_size[old_bin_num - 1]) {
1551 						/* truncation */
1552 						ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1553 						copy_size = use_copy_size ? MIN(size, copy_size) : size;
1554 						memcpy(ret, ptr, copy_size);
1555 						zend_mm_free_small(heap, ptr, old_bin_num);
1556 					} else {
1557 						/* reallocation in-place */
1558 						ret = ptr;
1559 					}
1560 				} else if (size <= ZEND_MM_MAX_SMALL_SIZE) {
1561 					/* small extension */
1562 
1563 #if ZEND_MM_STAT
1564 					do {
1565 						size_t orig_peak = heap->peak;
1566 #endif
1567 						ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1568 						copy_size = use_copy_size ? MIN(old_size, copy_size) : old_size;
1569 						memcpy(ret, ptr, copy_size);
1570 						zend_mm_free_small(heap, ptr, old_bin_num);
1571 #if ZEND_MM_STAT
1572 						heap->peak = MAX(orig_peak, heap->size);
1573 					} while (0);
1574 #endif
1575 				} else {
1576 					/* slow reallocation */
1577 					break;
1578 				}
1579 
1580 #if ZEND_DEBUG
1581 				dbg = zend_mm_get_debug_info(heap, ret);
1582 				dbg->size = real_size;
1583 				dbg->filename = __zend_filename;
1584 				dbg->orig_filename = __zend_orig_filename;
1585 				dbg->lineno = __zend_lineno;
1586 				dbg->orig_lineno = __zend_orig_lineno;
1587 #endif
1588 				return ret;
1589 			}  while (0);
1590 
1591 		} else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1592 			ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1593 			old_size = ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1594 			if (size > ZEND_MM_MAX_SMALL_SIZE && size <= ZEND_MM_MAX_LARGE_SIZE) {
1595 				new_size = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE);
1596 				if (new_size == old_size) {
1597 #if ZEND_DEBUG
1598 					dbg = zend_mm_get_debug_info(heap, ptr);
1599 					dbg->size = real_size;
1600 					dbg->filename = __zend_filename;
1601 					dbg->orig_filename = __zend_orig_filename;
1602 					dbg->lineno = __zend_lineno;
1603 					dbg->orig_lineno = __zend_orig_lineno;
1604 #endif
1605 					return ptr;
1606 				} else if (new_size < old_size) {
1607 					/* free tail pages */
1608 					int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1609 					int rest_pages_count = (int)((old_size - new_size) / ZEND_MM_PAGE_SIZE);
1610 
1611 #if ZEND_MM_STAT
1612 					heap->size -= rest_pages_count * ZEND_MM_PAGE_SIZE;
1613 #endif
1614 					chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1615 					chunk->free_pages += rest_pages_count;
1616 					zend_mm_bitset_reset_range(chunk->free_map, page_num + new_pages_count, rest_pages_count);
1617 #if ZEND_DEBUG
1618 					dbg = zend_mm_get_debug_info(heap, ptr);
1619 					dbg->size = real_size;
1620 					dbg->filename = __zend_filename;
1621 					dbg->orig_filename = __zend_orig_filename;
1622 					dbg->lineno = __zend_lineno;
1623 					dbg->orig_lineno = __zend_orig_lineno;
1624 #endif
1625 					return ptr;
1626 				} else /* if (new_size > old_size) */ {
1627 					int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1628 					int old_pages_count = (int)(old_size / ZEND_MM_PAGE_SIZE);
1629 
1630 					/* try to allocate tail pages after this block */
1631 					if (page_num + new_pages_count <= ZEND_MM_PAGES &&
1632 					    zend_mm_bitset_is_free_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count)) {
1633 #if ZEND_MM_STAT
1634 						do {
1635 							size_t size = heap->size + (new_size - old_size);
1636 							size_t peak = MAX(heap->peak, size);
1637 							heap->size = size;
1638 							heap->peak = peak;
1639 						} while (0);
1640 #endif
1641 						chunk->free_pages -= new_pages_count - old_pages_count;
1642 						zend_mm_bitset_set_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count);
1643 						chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1644 #if ZEND_DEBUG
1645 						dbg = zend_mm_get_debug_info(heap, ptr);
1646 						dbg->size = real_size;
1647 						dbg->filename = __zend_filename;
1648 						dbg->orig_filename = __zend_orig_filename;
1649 						dbg->lineno = __zend_lineno;
1650 						dbg->orig_lineno = __zend_orig_lineno;
1651 #endif
1652 						return ptr;
1653 					}
1654 				}
1655 			}
1656 		}
1657 #if ZEND_DEBUG
1658 		size = real_size;
1659 #endif
1660 	}
1661 
1662 	copy_size = MIN(old_size, copy_size);
1663 	return zend_mm_realloc_slow(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1664 }
1665 
1666 /*********************/
1667 /* Huge Runs (again) */
1668 /*********************/
1669 
1670 #if ZEND_DEBUG
1671 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1672 #else
1673 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1674 #endif
1675 {
1676 	zend_mm_huge_list *list = (zend_mm_huge_list*)zend_mm_alloc_heap(heap, sizeof(zend_mm_huge_list) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1677 	list->ptr = ptr;
1678 	list->size = size;
1679 	list->next = heap->huge_list;
1680 #if ZEND_DEBUG
1681 	list->dbg.size = dbg_size;
1682 	list->dbg.filename = __zend_filename;
1683 	list->dbg.orig_filename = __zend_orig_filename;
1684 	list->dbg.lineno = __zend_lineno;
1685 	list->dbg.orig_lineno = __zend_orig_lineno;
1686 #endif
1687 	heap->huge_list = list;
1688 }
1689 
1690 static size_t zend_mm_del_huge_block(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1691 {
1692 	zend_mm_huge_list *prev = NULL;
1693 	zend_mm_huge_list *list = heap->huge_list;
1694 	while (list != NULL) {
1695 		if (list->ptr == ptr) {
1696 			size_t size;
1697 
1698 			if (prev) {
1699 				prev->next = list->next;
1700 			} else {
1701 				heap->huge_list = list->next;
1702 			}
1703 			size = list->size;
1704 			zend_mm_free_heap(heap, list ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1705 			return size;
1706 		}
1707 		prev = list;
1708 		list = list->next;
1709 	}
1710 	ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1711 	return 0;
1712 }
1713 
1714 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1715 {
1716 	zend_mm_huge_list *list = heap->huge_list;
1717 	while (list != NULL) {
1718 		if (list->ptr == ptr) {
1719 			return list->size;
1720 		}
1721 		list = list->next;
1722 	}
1723 	ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1724 	return 0;
1725 }
1726 
1727 #if ZEND_DEBUG
1728 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1729 #else
1730 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1731 #endif
1732 {
1733 	zend_mm_huge_list *list = heap->huge_list;
1734 	while (list != NULL) {
1735 		if (list->ptr == ptr) {
1736 			list->size = size;
1737 #if ZEND_DEBUG
1738 			list->dbg.size = dbg_size;
1739 			list->dbg.filename = __zend_filename;
1740 			list->dbg.orig_filename = __zend_orig_filename;
1741 			list->dbg.lineno = __zend_lineno;
1742 			list->dbg.orig_lineno = __zend_orig_lineno;
1743 #endif
1744 			return;
1745 		}
1746 		list = list->next;
1747 	}
1748 }
1749 
1750 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1751 {
1752 #ifdef ZEND_WIN32
1753 	/* On Windows we don't have ability to extend huge blocks in-place.
1754 	 * We allocate them with 2MB size granularity, to avoid many
1755 	 * reallocations when they are extended by small pieces
1756 	 */
1757 	size_t alignment = MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE);
1758 #else
1759 	size_t alignment = REAL_PAGE_SIZE;
1760 #endif
1761 	size_t new_size = ZEND_MM_ALIGNED_SIZE_EX(size, alignment);
1762 	void *ptr;
1763 
1764 	if (UNEXPECTED(new_size < size)) {
1765 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", size, alignment);
1766 	}
1767 
1768 #if ZEND_MM_LIMIT
1769 	if (UNEXPECTED(new_size > heap->limit - heap->real_size)) {
1770 		if (zend_mm_gc(heap) && new_size <= heap->limit - heap->real_size) {
1771 			/* pass */
1772 		} else if (heap->overflow == 0) {
1773 #if ZEND_DEBUG
1774 			zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1775 #else
1776 			zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1777 #endif
1778 			return NULL;
1779 		}
1780 	}
1781 #endif
1782 	ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE);
1783 	if (UNEXPECTED(ptr == NULL)) {
1784 		/* insufficient memory */
1785 		if (zend_mm_gc(heap) &&
1786 		    (ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE)) != NULL) {
1787 			/* pass */
1788 		} else {
1789 #if !ZEND_MM_LIMIT
1790 			zend_mm_safe_error(heap, "Out of memory");
1791 #elif ZEND_DEBUG
1792 			zend_mm_safe_error(heap, "Out of memory (allocated %zu) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1793 #else
1794 			zend_mm_safe_error(heap, "Out of memory (allocated %zu) (tried to allocate %zu bytes)", heap->real_size, size);
1795 #endif
1796 			return NULL;
1797 		}
1798 	}
1799 #if ZEND_DEBUG
1800 	zend_mm_add_huge_block(heap, ptr, new_size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1801 #else
1802 	zend_mm_add_huge_block(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1803 #endif
1804 #if ZEND_MM_STAT
1805 	do {
1806 		size_t size = heap->real_size + new_size;
1807 		size_t peak = MAX(heap->real_peak, size);
1808 		heap->real_size = size;
1809 		heap->real_peak = peak;
1810 	} while (0);
1811 	do {
1812 		size_t size = heap->size + new_size;
1813 		size_t peak = MAX(heap->peak, size);
1814 		heap->size = size;
1815 		heap->peak = peak;
1816 	} while (0);
1817 #elif ZEND_MM_LIMIT
1818 	heap->real_size += new_size;
1819 #endif
1820 	return ptr;
1821 }
1822 
1823 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1824 {
1825 	size_t size;
1826 
1827 	ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE) == 0, "zend_mm_heap corrupted");
1828 	size = zend_mm_del_huge_block(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1829 	zend_mm_chunk_free(heap, ptr, size);
1830 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1831 	heap->real_size -= size;
1832 #endif
1833 #if ZEND_MM_STAT
1834 	heap->size -= size;
1835 #endif
1836 }
1837 
1838 /******************/
1839 /* Initialization */
1840 /******************/
1841 
1842 static zend_mm_heap *zend_mm_init(void)
1843 {
1844 	zend_mm_chunk *chunk = (zend_mm_chunk*)zend_mm_chunk_alloc_int(ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1845 	zend_mm_heap *heap;
1846 
1847 	if (UNEXPECTED(chunk == NULL)) {
1848 #if ZEND_MM_ERROR
1849 #ifdef _WIN32
1850 		stderr_last_error("Can't initialize heap");
1851 #else
1852 		fprintf(stderr, "\nCan't initialize heap: [%d] %s\n", errno, strerror(errno));
1853 #endif
1854 #endif
1855 		return NULL;
1856 	}
1857 	heap = &chunk->heap_slot;
1858 	chunk->heap = heap;
1859 	chunk->next = chunk;
1860 	chunk->prev = chunk;
1861 	chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1862 	chunk->free_tail = ZEND_MM_FIRST_PAGE;
1863 	chunk->num = 0;
1864 	chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
1865 	chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
1866 	heap->main_chunk = chunk;
1867 	heap->cached_chunks = NULL;
1868 	heap->chunks_count = 1;
1869 	heap->peak_chunks_count = 1;
1870 	heap->cached_chunks_count = 0;
1871 	heap->avg_chunks_count = 1.0;
1872 	heap->last_chunks_delete_boundary = 0;
1873 	heap->last_chunks_delete_count = 0;
1874 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1875 	heap->real_size = ZEND_MM_CHUNK_SIZE;
1876 #endif
1877 #if ZEND_MM_STAT
1878 	heap->real_peak = ZEND_MM_CHUNK_SIZE;
1879 	heap->size = 0;
1880 	heap->peak = 0;
1881 #endif
1882 #if ZEND_MM_LIMIT
1883 	heap->limit = ((size_t)Z_L(-1) >> (size_t)Z_L(1));
1884 	heap->overflow = 0;
1885 #endif
1886 #if ZEND_MM_CUSTOM
1887 	heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
1888 #endif
1889 #if ZEND_MM_STORAGE
1890 	heap->storage = NULL;
1891 #endif
1892 	heap->huge_list = NULL;
1893 	return heap;
1894 }
1895 
1896 ZEND_API size_t zend_mm_gc(zend_mm_heap *heap)
1897 {
1898 	zend_mm_free_slot *p, **q;
1899 	zend_mm_chunk *chunk;
1900 	size_t page_offset;
1901 	int page_num;
1902 	zend_mm_page_info info;
1903 	uint32_t i, free_counter;
1904 	int has_free_pages;
1905 	size_t collected = 0;
1906 
1907 #if ZEND_MM_CUSTOM
1908 	if (heap->use_custom_heap) {
1909 		return 0;
1910 	}
1911 #endif
1912 
1913 	for (i = 0; i < ZEND_MM_BINS; i++) {
1914 		has_free_pages = 0;
1915 		p = heap->free_slot[i];
1916 		while (p != NULL) {
1917 			chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
1918 			ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1919 			page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
1920 			ZEND_ASSERT(page_offset != 0);
1921 			page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1922 			info = chunk->map[page_num];
1923 			ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1924 			if (info & ZEND_MM_IS_LRUN) {
1925 				page_num -= ZEND_MM_NRUN_OFFSET(info);
1926 				info = chunk->map[page_num];
1927 				ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1928 				ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
1929 			}
1930 			ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
1931 			free_counter = ZEND_MM_SRUN_FREE_COUNTER(info) + 1;
1932 			if (free_counter == bin_elements[i]) {
1933 				has_free_pages = 1;
1934 			}
1935 			chunk->map[page_num] = ZEND_MM_SRUN_EX(i, free_counter);
1936 			p = p->next_free_slot;
1937 		}
1938 
1939 		if (!has_free_pages) {
1940 			continue;
1941 		}
1942 
1943 		q = &heap->free_slot[i];
1944 		p = *q;
1945 		while (p != NULL) {
1946 			chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
1947 			ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1948 			page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
1949 			ZEND_ASSERT(page_offset != 0);
1950 			page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1951 			info = chunk->map[page_num];
1952 			ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1953 			if (info & ZEND_MM_IS_LRUN) {
1954 				page_num -= ZEND_MM_NRUN_OFFSET(info);
1955 				info = chunk->map[page_num];
1956 				ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1957 				ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
1958 			}
1959 			ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
1960 			if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[i]) {
1961 				/* remove from cache */
1962 				p = p->next_free_slot;
1963 				*q = p;
1964 			} else {
1965 				q = &p->next_free_slot;
1966 				p = *q;
1967 			}
1968 		}
1969 	}
1970 
1971 	chunk = heap->main_chunk;
1972 	do {
1973 		i = ZEND_MM_FIRST_PAGE;
1974 		while (i < chunk->free_tail) {
1975 			if (zend_mm_bitset_is_set(chunk->free_map, i)) {
1976 				info = chunk->map[i];
1977 				if (info & ZEND_MM_IS_SRUN) {
1978 					int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1979 					int pages_count = bin_pages[bin_num];
1980 
1981 					if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[bin_num]) {
1982 						/* all elements are free */
1983 						zend_mm_free_pages_ex(heap, chunk, i, pages_count, 0);
1984 						collected += pages_count;
1985 					} else {
1986 						/* reset counter */
1987 						chunk->map[i] = ZEND_MM_SRUN(bin_num);
1988 					}
1989 					i += bin_pages[bin_num];
1990 				} else /* if (info & ZEND_MM_IS_LRUN) */ {
1991 					i += ZEND_MM_LRUN_PAGES(info);
1992 				}
1993 			} else {
1994 				i++;
1995 			}
1996 		}
1997 		if (chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
1998 			zend_mm_chunk *next_chunk = chunk->next;
1999 
2000 			zend_mm_delete_chunk(heap, chunk);
2001 			chunk = next_chunk;
2002 		} else {
2003 			chunk = chunk->next;
2004 		}
2005 	} while (chunk != heap->main_chunk);
2006 
2007 	return collected * ZEND_MM_PAGE_SIZE;
2008 }
2009 
2010 #if ZEND_DEBUG
2011 /******************/
2012 /* Leak detection */
2013 /******************/
2014 
2015 static zend_long zend_mm_find_leaks_small(zend_mm_chunk *p, uint32_t i, uint32_t j, zend_leak_info *leak)
2016 {
2017     int empty = 1;
2018 	zend_long count = 0;
2019 	int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2020 	zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * (j + 1) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2021 
2022 	while (j < bin_elements[bin_num]) {
2023 		if (dbg->size != 0) {
2024 			if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2025 				count++;
2026 				dbg->size = 0;
2027 				dbg->filename = NULL;
2028 				dbg->lineno = 0;
2029 			} else {
2030 				empty = 0;
2031 			}
2032 		}
2033 		j++;
2034 		dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2035 	}
2036 	if (empty) {
2037 		zend_mm_bitset_reset_range(p->free_map, i, bin_pages[bin_num]);
2038 	}
2039 	return count;
2040 }
2041 
2042 static zend_long zend_mm_find_leaks(zend_mm_heap *heap, zend_mm_chunk *p, uint32_t i, zend_leak_info *leak)
2043 {
2044 	zend_long count = 0;
2045 
2046 	do {
2047 		while (i < p->free_tail) {
2048 			if (zend_mm_bitset_is_set(p->free_map, i)) {
2049 				if (p->map[i] & ZEND_MM_IS_SRUN) {
2050 					int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2051 					count += zend_mm_find_leaks_small(p, i, 0, leak);
2052 					i += bin_pages[bin_num];
2053 				} else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2054 					int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2055 					zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2056 
2057 					if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2058 						count++;
2059 					}
2060 					zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2061 					i += pages_count;
2062 				}
2063 			} else {
2064 				i++;
2065 			}
2066 		}
2067 		p = p->next;
2068 		i = ZEND_MM_FIRST_PAGE;
2069 	} while (p != heap->main_chunk);
2070 	return count;
2071 }
2072 
2073 static zend_long zend_mm_find_leaks_huge(zend_mm_heap *heap, zend_mm_huge_list *list)
2074 {
2075 	zend_long count = 0;
2076 	zend_mm_huge_list *prev = list;
2077 	zend_mm_huge_list *p = list->next;
2078 
2079 	while (p) {
2080 		if (p->dbg.filename == list->dbg.filename && p->dbg.lineno == list->dbg.lineno) {
2081 			prev->next = p->next;
2082 			zend_mm_chunk_free(heap, p->ptr, p->size);
2083 			zend_mm_free_heap(heap, p, NULL, 0, NULL, 0);
2084 			count++;
2085 		} else {
2086 			prev = p;
2087 		}
2088 		p = prev->next;
2089 	}
2090 
2091 	return count;
2092 }
2093 
2094 static void zend_mm_check_leaks(zend_mm_heap *heap)
2095 {
2096 	zend_mm_huge_list *list;
2097 	zend_mm_chunk *p;
2098 	zend_leak_info leak;
2099 	zend_long repeated = 0;
2100 	uint32_t total = 0;
2101 	uint32_t i, j;
2102 
2103 	/* find leaked huge blocks and free them */
2104 	list = heap->huge_list;
2105 	while (list) {
2106 		zend_mm_huge_list *q = list;
2107 
2108 		leak.addr = list->ptr;
2109 		leak.size = list->dbg.size;
2110 		leak.filename = list->dbg.filename;
2111 		leak.orig_filename = list->dbg.orig_filename;
2112 		leak.lineno = list->dbg.lineno;
2113 		leak.orig_lineno = list->dbg.orig_lineno;
2114 
2115 		zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2116 		zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2117 		repeated = zend_mm_find_leaks_huge(heap, list);
2118 		total += 1 + repeated;
2119 		if (repeated) {
2120 			zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2121 		}
2122 
2123 		heap->huge_list = list = list->next;
2124 		zend_mm_chunk_free(heap, q->ptr, q->size);
2125 		zend_mm_free_heap(heap, q, NULL, 0, NULL, 0);
2126 	}
2127 
2128 	/* for each chunk */
2129 	p = heap->main_chunk;
2130 	do {
2131 		i = ZEND_MM_FIRST_PAGE;
2132 		while (i < p->free_tail) {
2133 			if (zend_mm_bitset_is_set(p->free_map, i)) {
2134 				if (p->map[i] & ZEND_MM_IS_SRUN) {
2135 					int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2136 					zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2137 
2138 					j = 0;
2139 					while (j < bin_elements[bin_num]) {
2140 						if (dbg->size != 0) {
2141 							leak.addr = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * j);
2142 							leak.size = dbg->size;
2143 							leak.filename = dbg->filename;
2144 							leak.orig_filename = dbg->orig_filename;
2145 							leak.lineno = dbg->lineno;
2146 							leak.orig_lineno = dbg->orig_lineno;
2147 
2148 							zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2149 							zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2150 
2151 							dbg->size = 0;
2152 							dbg->filename = NULL;
2153 							dbg->lineno = 0;
2154 
2155 							repeated = zend_mm_find_leaks_small(p, i, j + 1, &leak) +
2156 							           zend_mm_find_leaks(heap, p, i + bin_pages[bin_num], &leak);
2157 							total += 1 + repeated;
2158 							if (repeated) {
2159 								zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2160 							}
2161 						}
2162 						dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2163 						j++;
2164 					}
2165 					i += bin_pages[bin_num];
2166 				} else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2167 					int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2168 					zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2169 
2170 					leak.addr = (void*)((char*)p + ZEND_MM_PAGE_SIZE * i);
2171 					leak.size = dbg->size;
2172 					leak.filename = dbg->filename;
2173 					leak.orig_filename = dbg->orig_filename;
2174 					leak.lineno = dbg->lineno;
2175 					leak.orig_lineno = dbg->orig_lineno;
2176 
2177 					zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2178 					zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2179 
2180 					zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2181 
2182 					repeated = zend_mm_find_leaks(heap, p, i + pages_count, &leak);
2183 					total += 1 + repeated;
2184 					if (repeated) {
2185 						zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2186 					}
2187 					i += pages_count;
2188 				}
2189 			} else {
2190 				i++;
2191 			}
2192 		}
2193 		p = p->next;
2194 	} while (p != heap->main_chunk);
2195 	if (total) {
2196 		zend_message_dispatcher(ZMSG_MEMORY_LEAKS_GRAND_TOTAL, &total);
2197 	}
2198 }
2199 #endif
2200 
2201 #if ZEND_MM_CUSTOM
2202 static void *tracked_malloc(size_t size);
2203 static void tracked_free_all();
2204 #endif
2205 
2206 void zend_mm_shutdown(zend_mm_heap *heap, int full, int silent)
2207 {
2208 	zend_mm_chunk *p;
2209 	zend_mm_huge_list *list;
2210 
2211 #if ZEND_MM_CUSTOM
2212 	if (heap->use_custom_heap) {
2213 		if (heap->custom_heap.std._malloc == tracked_malloc) {
2214 			if (silent) {
2215 				tracked_free_all();
2216 			}
2217 			zend_hash_clean(heap->tracked_allocs);
2218 			if (full) {
2219 				zend_hash_destroy(heap->tracked_allocs);
2220 				free(heap->tracked_allocs);
2221 				/* Make sure the heap free below does not use tracked_free(). */
2222 				heap->custom_heap.std._free = free;
2223 			}
2224 		}
2225 
2226 		if (full) {
2227 			if (ZEND_DEBUG && heap->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2228 				heap->custom_heap.debug._free(heap ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC);
2229 			} else {
2230 				heap->custom_heap.std._free(heap);
2231 			}
2232 		}
2233 		return;
2234 	}
2235 #endif
2236 
2237 #if ZEND_DEBUG
2238 	if (!silent) {
2239 		zend_mm_check_leaks(heap);
2240 	}
2241 #endif
2242 
2243 	/* free huge blocks */
2244 	list = heap->huge_list;
2245 	heap->huge_list = NULL;
2246 	while (list) {
2247 		zend_mm_huge_list *q = list;
2248 		list = list->next;
2249 		zend_mm_chunk_free(heap, q->ptr, q->size);
2250 	}
2251 
2252 	/* move all chunks except of the first one into the cache */
2253 	p = heap->main_chunk->next;
2254 	while (p != heap->main_chunk) {
2255 		zend_mm_chunk *q = p->next;
2256 		p->next = heap->cached_chunks;
2257 		heap->cached_chunks = p;
2258 		p = q;
2259 		heap->chunks_count--;
2260 		heap->cached_chunks_count++;
2261 	}
2262 
2263 	if (full) {
2264 		/* free all cached chunks */
2265 		while (heap->cached_chunks) {
2266 			p = heap->cached_chunks;
2267 			heap->cached_chunks = p->next;
2268 			zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2269 		}
2270 		/* free the first chunk */
2271 		zend_mm_chunk_free(heap, heap->main_chunk, ZEND_MM_CHUNK_SIZE);
2272 	} else {
2273 		/* free some cached chunks to keep average count */
2274 		heap->avg_chunks_count = (heap->avg_chunks_count + (double)heap->peak_chunks_count) / 2.0;
2275 		while ((double)heap->cached_chunks_count + 0.9 > heap->avg_chunks_count &&
2276 		       heap->cached_chunks) {
2277 			p = heap->cached_chunks;
2278 			heap->cached_chunks = p->next;
2279 			zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2280 			heap->cached_chunks_count--;
2281 		}
2282 		/* clear cached chunks */
2283 		p = heap->cached_chunks;
2284 		while (p != NULL) {
2285 			zend_mm_chunk *q = p->next;
2286 			memset(p, 0, sizeof(zend_mm_chunk));
2287 			p->next = q;
2288 			p = q;
2289 		}
2290 
2291 		/* reinitialize the first chunk and heap */
2292 		p = heap->main_chunk;
2293 		p->heap = &p->heap_slot;
2294 		p->next = p;
2295 		p->prev = p;
2296 		p->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
2297 		p->free_tail = ZEND_MM_FIRST_PAGE;
2298 		p->num = 0;
2299 
2300 #if ZEND_MM_STAT
2301 		heap->size = heap->peak = 0;
2302 #endif
2303 		memset(heap->free_slot, 0, sizeof(heap->free_slot));
2304 #if ZEND_MM_STAT || ZEND_MM_LIMIT
2305 		heap->real_size = ZEND_MM_CHUNK_SIZE;
2306 #endif
2307 #if ZEND_MM_STAT
2308 		heap->real_peak = ZEND_MM_CHUNK_SIZE;
2309 #endif
2310 		heap->chunks_count = 1;
2311 		heap->peak_chunks_count = 1;
2312 		heap->last_chunks_delete_boundary = 0;
2313 		heap->last_chunks_delete_count = 0;
2314 
2315 		memset(p->free_map, 0, sizeof(p->free_map) + sizeof(p->map));
2316 		p->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
2317 		p->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
2318 	}
2319 }
2320 
2321 /**************/
2322 /* PUBLIC API */
2323 /**************/
2324 
2325 ZEND_API void* ZEND_FASTCALL _zend_mm_alloc(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2326 {
2327 	return zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2328 }
2329 
2330 ZEND_API void ZEND_FASTCALL _zend_mm_free(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2331 {
2332 	zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2333 }
2334 
2335 void* ZEND_FASTCALL _zend_mm_realloc(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2336 {
2337 	return zend_mm_realloc_heap(heap, ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2338 }
2339 
2340 void* ZEND_FASTCALL _zend_mm_realloc2(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2341 {
2342 	return zend_mm_realloc_heap(heap, ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2343 }
2344 
2345 ZEND_API size_t ZEND_FASTCALL _zend_mm_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2346 {
2347 	return zend_mm_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2348 }
2349 
2350 /**********************/
2351 /* Allocation Manager */
2352 /**********************/
2353 
2354 typedef struct _zend_alloc_globals {
2355 	zend_mm_heap *mm_heap;
2356 } zend_alloc_globals;
2357 
2358 #ifdef ZTS
2359 static int alloc_globals_id;
2360 static size_t alloc_globals_offset;
2361 # define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v)
2362 #else
2363 # define AG(v) (alloc_globals.v)
2364 static zend_alloc_globals alloc_globals;
2365 #endif
2366 
2367 ZEND_API int is_zend_mm(void)
2368 {
2369 #if ZEND_MM_CUSTOM
2370 	return !AG(mm_heap)->use_custom_heap;
2371 #else
2372 	return 1;
2373 #endif
2374 }
2375 
2376 ZEND_API int is_zend_ptr(const void *ptr)
2377 {
2378 #if ZEND_MM_CUSTOM
2379 	if (AG(mm_heap)->use_custom_heap) {
2380 		return 0;
2381 	}
2382 #endif
2383 
2384 	if (AG(mm_heap)->main_chunk) {
2385 		zend_mm_chunk *chunk = AG(mm_heap)->main_chunk;
2386 
2387 		do {
2388 			if (ptr >= (void*)chunk
2389 			 && ptr < (void*)((char*)chunk + ZEND_MM_CHUNK_SIZE)) {
2390 				return 1;
2391 			}
2392 			chunk = chunk->next;
2393 		} while (chunk != AG(mm_heap)->main_chunk);
2394 	}
2395 
2396 	if (AG(mm_heap)->huge_list) {
2397 		zend_mm_huge_list *block = AG(mm_heap)->huge_list;
2398 
2399 		do {
2400 			if (ptr >= (void*)block
2401 			 && ptr < (void*)((char*)block + block->size)) {
2402 				return 1;
2403 			}
2404 			block = block->next;
2405 		} while (block != AG(mm_heap)->huge_list);
2406 	}
2407 	return 0;
2408 }
2409 
2410 #if ZEND_MM_CUSTOM
2411 
2412 static ZEND_COLD void* ZEND_FASTCALL _malloc_custom(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2413 {
2414 	if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2415 		return AG(mm_heap)->custom_heap.debug._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2416 	} else {
2417 		return AG(mm_heap)->custom_heap.std._malloc(size);
2418 	}
2419 }
2420 
2421 static ZEND_COLD void ZEND_FASTCALL _efree_custom(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2422 {
2423 	if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2424 		AG(mm_heap)->custom_heap.debug._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2425 	} else {
2426 		AG(mm_heap)->custom_heap.std._free(ptr);
2427     }
2428 }
2429 
2430 static ZEND_COLD void* ZEND_FASTCALL _realloc_custom(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2431 {
2432 	if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2433 		return AG(mm_heap)->custom_heap.debug._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2434 	} else {
2435 		return AG(mm_heap)->custom_heap.std._realloc(ptr, size);
2436 	}
2437 }
2438 #endif
2439 
2440 #if !ZEND_DEBUG && defined(HAVE_BUILTIN_CONSTANT_P)
2441 #undef _emalloc
2442 
2443 #if ZEND_MM_CUSTOM
2444 # define ZEND_MM_CUSTOM_ALLOCATOR(size) do { \
2445 		if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2446 			return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2447 		} \
2448 	} while (0)
2449 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr) do { \
2450 		if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2451 			_efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2452 			return; \
2453 		} \
2454 	} while (0)
2455 #else
2456 # define ZEND_MM_CUSTOM_ALLOCATOR(size)
2457 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr)
2458 #endif
2459 
2460 # define _ZEND_BIN_ALLOCATOR(_num, _size, _elements, _pages, x, y) \
2461 	ZEND_API void* ZEND_FASTCALL _emalloc_ ## _size(void) { \
2462 		ZEND_MM_CUSTOM_ALLOCATOR(_size); \
2463 		return zend_mm_alloc_small(AG(mm_heap), _num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2464 	}
2465 
2466 ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR, x, y)
2467 
2468 ZEND_API void* ZEND_FASTCALL _emalloc_large(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2469 {
2470 	ZEND_MM_CUSTOM_ALLOCATOR(size);
2471 	return zend_mm_alloc_large_ex(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2472 }
2473 
2474 ZEND_API void* ZEND_FASTCALL _emalloc_huge(size_t size)
2475 {
2476 	ZEND_MM_CUSTOM_ALLOCATOR(size);
2477 	return zend_mm_alloc_huge(AG(mm_heap), size);
2478 }
2479 
2480 #if ZEND_DEBUG
2481 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2482 	ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2483 		ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2484 		{ \
2485 			size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); \
2486 			zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2487 			int page_num = page_offset / ZEND_MM_PAGE_SIZE; \
2488 			ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2489 			ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_SRUN); \
2490 			ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(chunk->map[page_num]) == _num); \
2491 			zend_mm_free_small(AG(mm_heap), ptr, _num); \
2492 		} \
2493 	}
2494 #else
2495 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2496 	ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2497 		ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2498 		{ \
2499 			zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2500 			ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2501 			zend_mm_free_small(AG(mm_heap), ptr, _num); \
2502 		} \
2503 	}
2504 #endif
2505 
2506 ZEND_MM_BINS_INFO(_ZEND_BIN_FREE, x, y)
2507 
2508 ZEND_API void ZEND_FASTCALL _efree_large(void *ptr, size_t size)
2509 {
2510 	ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2511 	{
2512 		size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
2513 		zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
2514 		int page_num = page_offset / ZEND_MM_PAGE_SIZE;
2515 		uint32_t pages_count = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE) / ZEND_MM_PAGE_SIZE;
2516 
2517 		ZEND_MM_CHECK(chunk->heap == AG(mm_heap) && ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
2518 		ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_LRUN);
2519 		ZEND_ASSERT(ZEND_MM_LRUN_PAGES(chunk->map[page_num]) == pages_count);
2520 		zend_mm_free_large(AG(mm_heap), chunk, page_num, pages_count);
2521 	}
2522 }
2523 
2524 ZEND_API void ZEND_FASTCALL _efree_huge(void *ptr, size_t size)
2525 {
2526 
2527 	ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2528 	zend_mm_free_huge(AG(mm_heap), ptr);
2529 }
2530 #endif
2531 
2532 ZEND_API void* ZEND_FASTCALL _emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2533 {
2534 #if ZEND_MM_CUSTOM
2535 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2536 		return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2537 	}
2538 #endif
2539 	return zend_mm_alloc_heap(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2540 }
2541 
2542 ZEND_API void ZEND_FASTCALL _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2543 {
2544 #if ZEND_MM_CUSTOM
2545 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2546 		_efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2547 		return;
2548 	}
2549 #endif
2550 	zend_mm_free_heap(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2551 }
2552 
2553 ZEND_API void* ZEND_FASTCALL _erealloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2554 {
2555 #if ZEND_MM_CUSTOM
2556 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2557 		return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2558 	}
2559 #endif
2560 	return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2561 }
2562 
2563 ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2564 {
2565 #if ZEND_MM_CUSTOM
2566 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2567 		return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2568 	}
2569 #endif
2570 	return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2571 }
2572 
2573 ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2574 {
2575 #if ZEND_MM_CUSTOM
2576 	if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2577 		return 0;
2578 	}
2579 #endif
2580 	return zend_mm_size(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2581 }
2582 
2583 ZEND_API void* ZEND_FASTCALL _safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2584 {
2585 	return _emalloc(zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2586 }
2587 
2588 ZEND_API void* ZEND_FASTCALL _safe_malloc(size_t nmemb, size_t size, size_t offset)
2589 {
2590 	return pemalloc(zend_safe_address_guarded(nmemb, size, offset), 1);
2591 }
2592 
2593 ZEND_API void* ZEND_FASTCALL _safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2594 {
2595 	return _erealloc(ptr, zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2596 }
2597 
2598 ZEND_API void* ZEND_FASTCALL _safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset)
2599 {
2600 	return perealloc(ptr, zend_safe_address_guarded(nmemb, size, offset), 1);
2601 }
2602 
2603 ZEND_API void* ZEND_FASTCALL _ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2604 {
2605 	void *p;
2606 
2607 	size = zend_safe_address_guarded(nmemb, size, 0);
2608 	p = _emalloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2609 	memset(p, 0, size);
2610 	return p;
2611 }
2612 
2613 ZEND_API char* ZEND_FASTCALL _estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2614 {
2615 	size_t length;
2616 	char *p;
2617 
2618 	length = strlen(s);
2619 	if (UNEXPECTED(length + 1 == 0)) {
2620 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2621 	}
2622 	p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2623 	memcpy(p, s, length+1);
2624 	return p;
2625 }
2626 
2627 ZEND_API char* ZEND_FASTCALL _estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2628 {
2629 	char *p;
2630 
2631 	if (UNEXPECTED(length + 1 == 0)) {
2632 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2633 	}
2634 	p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2635 	memcpy(p, s, length);
2636 	p[length] = 0;
2637 	return p;
2638 }
2639 
2640 
2641 ZEND_API char* ZEND_FASTCALL zend_strndup(const char *s, size_t length)
2642 {
2643 	char *p;
2644 
2645 	if (UNEXPECTED(length + 1 == 0)) {
2646 		zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2647 	}
2648 	p = (char *) malloc(length + 1);
2649 	if (UNEXPECTED(p == NULL)) {
2650 		return p;
2651 	}
2652 	if (EXPECTED(length)) {
2653 		memcpy(p, s, length);
2654 	}
2655 	p[length] = 0;
2656 	return p;
2657 }
2658 
2659 
2660 ZEND_API int zend_set_memory_limit(size_t memory_limit)
2661 {
2662 #if ZEND_MM_LIMIT
2663 	AG(mm_heap)->limit = (memory_limit >= ZEND_MM_CHUNK_SIZE) ? memory_limit : ZEND_MM_CHUNK_SIZE;
2664 #endif
2665 	return SUCCESS;
2666 }
2667 
2668 ZEND_API size_t zend_memory_usage(int real_usage)
2669 {
2670 #if ZEND_MM_STAT
2671 	if (real_usage) {
2672 		return AG(mm_heap)->real_size;
2673 	} else {
2674 		size_t usage = AG(mm_heap)->size;
2675 		return usage;
2676 	}
2677 #endif
2678 	return 0;
2679 }
2680 
2681 ZEND_API size_t zend_memory_peak_usage(int real_usage)
2682 {
2683 #if ZEND_MM_STAT
2684 	if (real_usage) {
2685 		return AG(mm_heap)->real_peak;
2686 	} else {
2687 		return AG(mm_heap)->peak;
2688 	}
2689 #endif
2690 	return 0;
2691 }
2692 
2693 ZEND_API void shutdown_memory_manager(int silent, int full_shutdown)
2694 {
2695 	zend_mm_shutdown(AG(mm_heap), full_shutdown, silent);
2696 }
2697 
2698 #if ZEND_MM_CUSTOM
2699 static void *tracked_malloc(size_t size)
2700 {
2701 	void *ptr = __zend_malloc(size);
2702 	zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2703 	ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr);
2704 	zend_hash_index_add_empty_element(AG(mm_heap)->tracked_allocs, h);
2705 	return ptr;
2706 }
2707 
2708 static void tracked_free(void *ptr) {
2709 	zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2710 	zend_hash_index_del(AG(mm_heap)->tracked_allocs, h);
2711 	free(ptr);
2712 }
2713 
2714 static void *tracked_realloc(void *ptr, size_t new_size) {
2715 	zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2716 	zend_hash_index_del(AG(mm_heap)->tracked_allocs, h);
2717 	ptr = __zend_realloc(ptr, new_size);
2718 	h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2719 	ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr);
2720 	zend_hash_index_add_empty_element(AG(mm_heap)->tracked_allocs, h);
2721 	return ptr;
2722 }
2723 
2724 static void tracked_free_all() {
2725 	HashTable *tracked_allocs = AG(mm_heap)->tracked_allocs;
2726 	zend_ulong h;
2727 	ZEND_HASH_FOREACH_NUM_KEY(tracked_allocs, h) {
2728 		void *ptr = (void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2);
2729 		free(ptr);
2730 	} ZEND_HASH_FOREACH_END();
2731 }
2732 #endif
2733 
2734 static void alloc_globals_ctor(zend_alloc_globals *alloc_globals)
2735 {
2736 	char *tmp;
2737 
2738 #if ZEND_MM_CUSTOM
2739 	tmp = getenv("USE_ZEND_ALLOC");
2740 	if (tmp && !zend_atoi(tmp, 0)) {
2741 		zend_bool tracked = (tmp = getenv("USE_TRACKED_ALLOC")) && zend_atoi(tmp, 0);
2742 		zend_mm_heap *mm_heap = alloc_globals->mm_heap = malloc(sizeof(zend_mm_heap));
2743 		memset(mm_heap, 0, sizeof(zend_mm_heap));
2744 		mm_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2745 		if (!tracked) {
2746 			/* Use system allocator. */
2747 			mm_heap->custom_heap.std._malloc = __zend_malloc;
2748 			mm_heap->custom_heap.std._free = free;
2749 			mm_heap->custom_heap.std._realloc = __zend_realloc;
2750 		} else {
2751 			/* Use system allocator and track allocations for auto-free. */
2752 			mm_heap->custom_heap.std._malloc = tracked_malloc;
2753 			mm_heap->custom_heap.std._free = tracked_free;
2754 			mm_heap->custom_heap.std._realloc = tracked_realloc;
2755 			mm_heap->tracked_allocs = malloc(sizeof(HashTable));
2756 			zend_hash_init(mm_heap->tracked_allocs, 1024, NULL, NULL, 1);
2757 		}
2758 		return;
2759 	}
2760 #endif
2761 
2762 	tmp = getenv("USE_ZEND_ALLOC_HUGE_PAGES");
2763 	if (tmp && zend_atoi(tmp, 0)) {
2764 		zend_mm_use_huge_pages = 1;
2765 	}
2766 	alloc_globals->mm_heap = zend_mm_init();
2767 }
2768 
2769 #ifdef ZTS
2770 static void alloc_globals_dtor(zend_alloc_globals *alloc_globals)
2771 {
2772 	zend_mm_shutdown(alloc_globals->mm_heap, 1, 1);
2773 }
2774 #endif
2775 
2776 ZEND_API void start_memory_manager(void)
2777 {
2778 #ifdef ZTS
2779 	ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
2780 #else
2781 	alloc_globals_ctor(&alloc_globals);
2782 #endif
2783 #ifndef _WIN32
2784 #  if defined(_SC_PAGESIZE)
2785 	REAL_PAGE_SIZE = sysconf(_SC_PAGESIZE);
2786 #  elif defined(_SC_PAGE_SIZE)
2787 	REAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
2788 #  endif
2789 #endif
2790 }
2791 
2792 ZEND_API zend_mm_heap *zend_mm_set_heap(zend_mm_heap *new_heap)
2793 {
2794 	zend_mm_heap *old_heap;
2795 
2796 	old_heap = AG(mm_heap);
2797 	AG(mm_heap) = (zend_mm_heap*)new_heap;
2798 	return (zend_mm_heap*)old_heap;
2799 }
2800 
2801 ZEND_API zend_mm_heap *zend_mm_get_heap(void)
2802 {
2803 	return AG(mm_heap);
2804 }
2805 
2806 ZEND_API int zend_mm_is_custom_heap(zend_mm_heap *new_heap)
2807 {
2808 #if ZEND_MM_CUSTOM
2809 	return AG(mm_heap)->use_custom_heap;
2810 #else
2811 	return 0;
2812 #endif
2813 }
2814 
2815 ZEND_API void zend_mm_set_custom_handlers(zend_mm_heap *heap,
2816                                           void* (*_malloc)(size_t),
2817                                           void  (*_free)(void*),
2818                                           void* (*_realloc)(void*, size_t))
2819 {
2820 #if ZEND_MM_CUSTOM
2821 	zend_mm_heap *_heap = (zend_mm_heap*)heap;
2822 
2823 	if (!_malloc && !_free && !_realloc) {
2824 		_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
2825 	} else {
2826 		_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2827 		_heap->custom_heap.std._malloc = _malloc;
2828 		_heap->custom_heap.std._free = _free;
2829 		_heap->custom_heap.std._realloc = _realloc;
2830 	}
2831 #endif
2832 }
2833 
2834 ZEND_API void zend_mm_get_custom_handlers(zend_mm_heap *heap,
2835                                           void* (**_malloc)(size_t),
2836                                           void  (**_free)(void*),
2837                                           void* (**_realloc)(void*, size_t))
2838 {
2839 #if ZEND_MM_CUSTOM
2840 	zend_mm_heap *_heap = (zend_mm_heap*)heap;
2841 
2842 	if (heap->use_custom_heap) {
2843 		*_malloc = _heap->custom_heap.std._malloc;
2844 		*_free = _heap->custom_heap.std._free;
2845 		*_realloc = _heap->custom_heap.std._realloc;
2846 	} else {
2847 		*_malloc = NULL;
2848 		*_free = NULL;
2849 		*_realloc = NULL;
2850 	}
2851 #else
2852 	*_malloc = NULL;
2853 	*_free = NULL;
2854 	*_realloc = NULL;
2855 #endif
2856 }
2857 
2858 #if ZEND_DEBUG
2859 ZEND_API void zend_mm_set_custom_debug_handlers(zend_mm_heap *heap,
2860                                           void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2861                                           void  (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2862                                           void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC))
2863 {
2864 #if ZEND_MM_CUSTOM
2865 	zend_mm_heap *_heap = (zend_mm_heap*)heap;
2866 
2867 	_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_DEBUG;
2868 	_heap->custom_heap.debug._malloc = _malloc;
2869 	_heap->custom_heap.debug._free = _free;
2870 	_heap->custom_heap.debug._realloc = _realloc;
2871 #endif
2872 }
2873 #endif
2874 
2875 ZEND_API zend_mm_storage *zend_mm_get_storage(zend_mm_heap *heap)
2876 {
2877 #if ZEND_MM_STORAGE
2878 	return heap->storage;
2879 #else
2880 	return NULL
2881 #endif
2882 }
2883 
2884 ZEND_API zend_mm_heap *zend_mm_startup(void)
2885 {
2886 	return zend_mm_init();
2887 }
2888 
2889 ZEND_API zend_mm_heap *zend_mm_startup_ex(const zend_mm_handlers *handlers, void *data, size_t data_size)
2890 {
2891 #if ZEND_MM_STORAGE
2892 	zend_mm_storage tmp_storage, *storage;
2893 	zend_mm_chunk *chunk;
2894 	zend_mm_heap *heap;
2895 
2896 	memcpy((zend_mm_handlers*)&tmp_storage.handlers, handlers, sizeof(zend_mm_handlers));
2897 	tmp_storage.data = data;
2898 	chunk = (zend_mm_chunk*)handlers->chunk_alloc(&tmp_storage, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
2899 	if (UNEXPECTED(chunk == NULL)) {
2900 #if ZEND_MM_ERROR
2901 #ifdef _WIN32
2902 		stderr_last_error("Can't initialize heap");
2903 #else
2904 		fprintf(stderr, "\nCan't initialize heap: [%d] %s\n", errno, strerror(errno));
2905 #endif
2906 #endif
2907 		return NULL;
2908 	}
2909 	heap = &chunk->heap_slot;
2910 	chunk->heap = heap;
2911 	chunk->next = chunk;
2912 	chunk->prev = chunk;
2913 	chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
2914 	chunk->free_tail = ZEND_MM_FIRST_PAGE;
2915 	chunk->num = 0;
2916 	chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
2917 	chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
2918 	heap->main_chunk = chunk;
2919 	heap->cached_chunks = NULL;
2920 	heap->chunks_count = 1;
2921 	heap->peak_chunks_count = 1;
2922 	heap->cached_chunks_count = 0;
2923 	heap->avg_chunks_count = 1.0;
2924 	heap->last_chunks_delete_boundary = 0;
2925 	heap->last_chunks_delete_count = 0;
2926 #if ZEND_MM_STAT || ZEND_MM_LIMIT
2927 	heap->real_size = ZEND_MM_CHUNK_SIZE;
2928 #endif
2929 #if ZEND_MM_STAT
2930 	heap->real_peak = ZEND_MM_CHUNK_SIZE;
2931 	heap->size = 0;
2932 	heap->peak = 0;
2933 #endif
2934 #if ZEND_MM_LIMIT
2935 	heap->limit = (Z_L(-1) >> Z_L(1));
2936 	heap->overflow = 0;
2937 #endif
2938 #if ZEND_MM_CUSTOM
2939 	heap->use_custom_heap = 0;
2940 #endif
2941 	heap->storage = &tmp_storage;
2942 	heap->huge_list = NULL;
2943 	memset(heap->free_slot, 0, sizeof(heap->free_slot));
2944 	storage = _zend_mm_alloc(heap, sizeof(zend_mm_storage) + data_size ZEND_FILE_LINE_CC ZEND_FILE_LINE_CC);
2945 	if (!storage) {
2946 		handlers->chunk_free(&tmp_storage, chunk, ZEND_MM_CHUNK_SIZE);
2947 #if ZEND_MM_ERROR
2948 #ifdef _WIN32
2949 		stderr_last_error("Can't initialize heap");
2950 #else
2951 		fprintf(stderr, "\nCan't initialize heap: [%d] %s\n", errno, strerror(errno));
2952 #endif
2953 #endif
2954 		return NULL;
2955 	}
2956 	memcpy(storage, &tmp_storage, sizeof(zend_mm_storage));
2957 	if (data) {
2958 		storage->dat