1 /*
2 +----------------------------------------------------------------------+
3 | Zend Engine |
4 +----------------------------------------------------------------------+
5 | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 2.00 of the Zend license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | http://www.zend.com/license/2_00.txt. |
11 | If you did not receive a copy of the Zend license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@zend.com so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Andi Gutmans <andi@php.net> |
16 | Zeev Suraski <zeev@php.net> |
17 | Dmitry Stogov <dmitry@php.net> |
18 +----------------------------------------------------------------------+
19 */
20
21 /*
22 * zend_alloc is designed to be a modern CPU cache friendly memory manager
23 * for PHP. Most ideas are taken from jemalloc and tcmalloc implementations.
24 *
25 * All allocations are split into 3 categories:
26 *
27 * Huge - the size is greater than CHUNK size (~2M by default), allocation is
28 * performed using mmap(). The result is aligned on 2M boundary.
29 *
30 * Large - a number of 4096K pages inside a CHUNK. Large blocks
31 * are always aligned on page boundary.
32 *
33 * Small - less than 3/4 of page size. Small sizes are rounded up to nearest
34 * greater predefined small size (there are 30 predefined sizes:
35 * 8, 16, 24, 32, ... 3072). Small blocks are allocated from
36 * RUNs. Each RUN is allocated as a single or few following pages.
37 * Allocation inside RUNs implemented using linked list of free
38 * elements. The result is aligned to 8 bytes.
39 *
40 * zend_alloc allocates memory from OS by CHUNKs, these CHUNKs and huge memory
41 * blocks are always aligned to CHUNK boundary. So it's very easy to determine
42 * the CHUNK owning the certain pointer. Regular CHUNKs reserve a single
43 * page at start for special purpose. It contains bitset of free pages,
44 * few bitset for available runs of predefined small sizes, map of pages that
45 * keeps information about usage of each page in this CHUNK, etc.
46 *
47 * zend_alloc provides familiar emalloc/efree/erealloc API, but in addition it
48 * provides specialized and optimized routines to allocate blocks of predefined
49 * sizes (e.g. emalloc_2(), emallc_4(), ..., emalloc_large(), etc)
50 * The library uses C preprocessor tricks that substitute calls to emalloc()
51 * with more specialized routines when the requested size is known.
52 */
53
54 #include "zend.h"
55 #include "zend_alloc.h"
56 #include "zend_globals.h"
57 #include "zend_operators.h"
58 #include "zend_multiply.h"
59 #include "zend_bitset.h"
60 #include <signal.h>
61
62 #ifdef HAVE_UNISTD_H
63 # include <unistd.h>
64 #endif
65
66 #ifdef ZEND_WIN32
67 # include <wincrypt.h>
68 # include <process.h>
69 # include "win32/winutil.h"
70 #endif
71
72 #include <stdio.h>
73 #include <stdlib.h>
74 #include <string.h>
75
76 #include <sys/types.h>
77 #include <sys/stat.h>
78 #include <limits.h>
79 #include <fcntl.h>
80 #include <errno.h>
81
82 #ifndef _WIN32
83 # include <sys/mman.h>
84 # ifndef MAP_ANON
85 # ifdef MAP_ANONYMOUS
86 # define MAP_ANON MAP_ANONYMOUS
87 # endif
88 # endif
89 # ifndef MAP_FAILED
90 # define MAP_FAILED ((void*)-1)
91 # endif
92 # ifndef MAP_POPULATE
93 # define MAP_POPULATE 0
94 # endif
95 # if defined(_SC_PAGESIZE) || (_SC_PAGE_SIZE)
96 # define REAL_PAGE_SIZE _real_page_size
97 static size_t _real_page_size = ZEND_MM_PAGE_SIZE;
98 # endif
99 # ifdef MAP_ALIGNED_SUPER
100 # define MAP_HUGETLB MAP_ALIGNED_SUPER
101 # endif
102 #endif
103
104 #ifndef REAL_PAGE_SIZE
105 # define REAL_PAGE_SIZE ZEND_MM_PAGE_SIZE
106 #endif
107
108 /* NetBSD has an mremap() function with a signature that is incompatible with Linux (WTF?),
109 * so pretend it doesn't exist. */
110 #ifndef __linux__
111 # undef HAVE_MREMAP
112 #endif
113
114 #ifndef __APPLE__
115 # define ZEND_MM_FD -1
116 #else
117 /* Mac allows to track anonymous page via vmmap per TAG id.
118 * user land applications are allowed to take from 240 to 255.
119 */
120 # define ZEND_MM_FD (250<<24)
121 #endif
122
123 #ifndef ZEND_MM_STAT
124 # define ZEND_MM_STAT 1 /* track current and peak memory usage */
125 #endif
126 #ifndef ZEND_MM_LIMIT
127 # define ZEND_MM_LIMIT 1 /* support for user-defined memory limit */
128 #endif
129 #ifndef ZEND_MM_CUSTOM
130 # define ZEND_MM_CUSTOM 1 /* support for custom memory allocator */
131 /* USE_ZEND_ALLOC=0 may switch to system malloc() */
132 #endif
133 #ifndef ZEND_MM_STORAGE
134 # define ZEND_MM_STORAGE 1 /* support for custom memory storage */
135 #endif
136 #ifndef ZEND_MM_ERROR
137 # define ZEND_MM_ERROR 1 /* report system errors */
138 #endif
139
140 #ifndef ZEND_MM_CHECK
141 # define ZEND_MM_CHECK(condition, message) do { \
142 if (UNEXPECTED(!(condition))) { \
143 zend_mm_panic(message); \
144 } \
145 } while (0)
146 #endif
147
148 typedef uint32_t zend_mm_page_info; /* 4-byte integer */
149 typedef zend_ulong zend_mm_bitset; /* 4-byte or 8-byte integer */
150
151 #define ZEND_MM_ALIGNED_OFFSET(size, alignment) \
152 (((size_t)(size)) & ((alignment) - 1))
153 #define ZEND_MM_ALIGNED_BASE(size, alignment) \
154 (((size_t)(size)) & ~((alignment) - 1))
155 #define ZEND_MM_SIZE_TO_NUM(size, alignment) \
156 (((size_t)(size) + ((alignment) - 1)) / (alignment))
157
158 #define ZEND_MM_BITSET_LEN (sizeof(zend_mm_bitset) * 8) /* 32 or 64 */
159 #define ZEND_MM_PAGE_MAP_LEN (ZEND_MM_PAGES / ZEND_MM_BITSET_LEN) /* 16 or 8 */
160
161 typedef zend_mm_bitset zend_mm_page_map[ZEND_MM_PAGE_MAP_LEN]; /* 64B */
162
163 #define ZEND_MM_IS_FRUN 0x00000000
164 #define ZEND_MM_IS_LRUN 0x40000000
165 #define ZEND_MM_IS_SRUN 0x80000000
166
167 #define ZEND_MM_LRUN_PAGES_MASK 0x000003ff
168 #define ZEND_MM_LRUN_PAGES_OFFSET 0
169
170 #define ZEND_MM_SRUN_BIN_NUM_MASK 0x0000001f
171 #define ZEND_MM_SRUN_BIN_NUM_OFFSET 0
172
173 #define ZEND_MM_SRUN_FREE_COUNTER_MASK 0x01ff0000
174 #define ZEND_MM_SRUN_FREE_COUNTER_OFFSET 16
175
176 #define ZEND_MM_NRUN_OFFSET_MASK 0x01ff0000
177 #define ZEND_MM_NRUN_OFFSET_OFFSET 16
178
179 #define ZEND_MM_LRUN_PAGES(info) (((info) & ZEND_MM_LRUN_PAGES_MASK) >> ZEND_MM_LRUN_PAGES_OFFSET)
180 #define ZEND_MM_SRUN_BIN_NUM(info) (((info) & ZEND_MM_SRUN_BIN_NUM_MASK) >> ZEND_MM_SRUN_BIN_NUM_OFFSET)
181 #define ZEND_MM_SRUN_FREE_COUNTER(info) (((info) & ZEND_MM_SRUN_FREE_COUNTER_MASK) >> ZEND_MM_SRUN_FREE_COUNTER_OFFSET)
182 #define ZEND_MM_NRUN_OFFSET(info) (((info) & ZEND_MM_NRUN_OFFSET_MASK) >> ZEND_MM_NRUN_OFFSET_OFFSET)
183
184 #define ZEND_MM_FRUN() ZEND_MM_IS_FRUN
185 #define ZEND_MM_LRUN(count) (ZEND_MM_IS_LRUN | ((count) << ZEND_MM_LRUN_PAGES_OFFSET))
186 #define ZEND_MM_SRUN(bin_num) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET))
187 #define ZEND_MM_SRUN_EX(bin_num, count) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((count) << ZEND_MM_SRUN_FREE_COUNTER_OFFSET))
188 #define ZEND_MM_NRUN(bin_num, offset) (ZEND_MM_IS_SRUN | ZEND_MM_IS_LRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((offset) << ZEND_MM_NRUN_OFFSET_OFFSET))
189
190 #define ZEND_MM_BINS 30
191
192 typedef struct _zend_mm_page zend_mm_page;
193 typedef struct _zend_mm_bin zend_mm_bin;
194 typedef struct _zend_mm_free_slot zend_mm_free_slot;
195 typedef struct _zend_mm_chunk zend_mm_chunk;
196 typedef struct _zend_mm_huge_list zend_mm_huge_list;
197
198 int zend_mm_use_huge_pages = 0;
199
200 /*
201 * Memory is retrieved from OS by chunks of fixed size 2MB.
202 * Inside chunk it's managed by pages of fixed size 4096B.
203 * So each chunk consists from 512 pages.
204 * The first page of each chunk is reserved for chunk header.
205 * It contains service information about all pages.
206 *
207 * free_pages - current number of free pages in this chunk
208 *
209 * free_tail - number of continuous free pages at the end of chunk
210 *
211 * free_map - bitset (a bit for each page). The bit is set if the corresponding
212 * page is allocated. Allocator for "lage sizes" may easily find a
213 * free page (or a continuous number of pages) searching for zero
214 * bits.
215 *
216 * map - contains service information for each page. (32-bits for each
217 * page).
218 * usage:
219 * (2 bits)
220 * FRUN - free page,
221 * LRUN - first page of "large" allocation
222 * SRUN - first page of a bin used for "small" allocation
223 *
224 * lrun_pages:
225 * (10 bits) number of allocated pages
226 *
227 * srun_bin_num:
228 * (5 bits) bin number (e.g. 0 for sizes 0-2, 1 for 3-4,
229 * 2 for 5-8, 3 for 9-16 etc) see zend_alloc_sizes.h
230 */
231
232 struct _zend_mm_heap {
233 #if ZEND_MM_CUSTOM
234 int use_custom_heap;
235 #endif
236 #if ZEND_MM_STORAGE
237 zend_mm_storage *storage;
238 #endif
239 #if ZEND_MM_STAT
240 size_t size; /* current memory usage */
241 size_t peak; /* peak memory usage */
242 #endif
243 zend_mm_free_slot *free_slot[ZEND_MM_BINS]; /* free lists for small sizes */
244 #if ZEND_MM_STAT || ZEND_MM_LIMIT
245 size_t real_size; /* current size of allocated pages */
246 #endif
247 #if ZEND_MM_STAT
248 size_t real_peak; /* peak size of allocated pages */
249 #endif
250 #if ZEND_MM_LIMIT
251 size_t limit; /* memory limit */
252 int overflow; /* memory overflow flag */
253 #endif
254
255 zend_mm_huge_list *huge_list; /* list of huge allocated blocks */
256
257 zend_mm_chunk *main_chunk;
258 zend_mm_chunk *cached_chunks; /* list of unused chunks */
259 int chunks_count; /* number of allocated chunks */
260 int peak_chunks_count; /* peak number of allocated chunks for current request */
261 int cached_chunks_count; /* number of cached chunks */
262 double avg_chunks_count; /* average number of chunks allocated per request */
263 int last_chunks_delete_boundary; /* numer of chunks after last deletion */
264 int last_chunks_delete_count; /* number of deletion over the last boundary */
265 #if ZEND_MM_CUSTOM
266 union {
267 struct {
268 void *(*_malloc)(size_t);
269 void (*_free)(void*);
270 void *(*_realloc)(void*, size_t);
271 } std;
272 struct {
273 void *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
274 void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
275 void *(*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
276 } debug;
277 } custom_heap;
278 HashTable *tracked_allocs;
279 #endif
280 };
281
282 struct _zend_mm_chunk {
283 zend_mm_heap *heap;
284 zend_mm_chunk *next;
285 zend_mm_chunk *prev;
286 uint32_t free_pages; /* number of free pages */
287 uint32_t free_tail; /* number of free pages at the end of chunk */
288 uint32_t num;
289 char reserve[64 - (sizeof(void*) * 3 + sizeof(uint32_t) * 3)];
290 zend_mm_heap heap_slot; /* used only in main chunk */
291 zend_mm_page_map free_map; /* 512 bits or 64 bytes */
292 zend_mm_page_info map[ZEND_MM_PAGES]; /* 2 KB = 512 * 4 */
293 };
294
295 struct _zend_mm_page {
296 char bytes[ZEND_MM_PAGE_SIZE];
297 };
298
299 /*
300 * bin - is one or few continuous pages (up to 8) used for allocation of
301 * a particular "small size".
302 */
303 struct _zend_mm_bin {
304 char bytes[ZEND_MM_PAGE_SIZE * 8];
305 };
306
307 struct _zend_mm_free_slot {
308 zend_mm_free_slot *next_free_slot;
309 };
310
311 struct _zend_mm_huge_list {
312 void *ptr;
313 size_t size;
314 zend_mm_huge_list *next;
315 #if ZEND_DEBUG
316 zend_mm_debug_info dbg;
317 #endif
318 };
319
320 #define ZEND_MM_PAGE_ADDR(chunk, page_num) \
321 ((void*)(((zend_mm_page*)(chunk)) + (page_num)))
322
323 #define _BIN_DATA_SIZE(num, size, elements, pages, x, y) size,
324 static const uint32_t bin_data_size[] = {
325 ZEND_MM_BINS_INFO(_BIN_DATA_SIZE, x, y)
326 };
327
328 #define _BIN_DATA_ELEMENTS(num, size, elements, pages, x, y) elements,
329 static const uint32_t bin_elements[] = {
330 ZEND_MM_BINS_INFO(_BIN_DATA_ELEMENTS, x, y)
331 };
332
333 #define _BIN_DATA_PAGES(num, size, elements, pages, x, y) pages,
334 static const uint32_t bin_pages[] = {
335 ZEND_MM_BINS_INFO(_BIN_DATA_PAGES, x, y)
336 };
337
338 #if ZEND_DEBUG
zend_debug_alloc_output(char * format,...)339 ZEND_COLD void zend_debug_alloc_output(char *format, ...)
340 {
341 char output_buf[256];
342 va_list args;
343
344 va_start(args, format);
345 vsprintf(output_buf, format, args);
346 va_end(args);
347
348 #ifdef ZEND_WIN32
349 OutputDebugString(output_buf);
350 #else
351 fprintf(stderr, "%s", output_buf);
352 #endif
353 }
354 #endif
355
zend_mm_panic(const char * message)356 static ZEND_COLD ZEND_NORETURN void zend_mm_panic(const char *message)
357 {
358 fprintf(stderr, "%s\n", message);
359 /* See http://support.microsoft.com/kb/190351 */
360 #ifdef ZEND_WIN32
361 fflush(stderr);
362 #endif
363 #if ZEND_DEBUG && defined(HAVE_KILL) && defined(HAVE_GETPID)
364 kill(getpid(), SIGSEGV);
365 #endif
366 exit(1);
367 }
368
zend_mm_safe_error(zend_mm_heap * heap,const char * format,size_t limit,const char * filename,uint32_t lineno,size_t size)369 static ZEND_COLD ZEND_NORETURN void zend_mm_safe_error(zend_mm_heap *heap,
370 const char *format,
371 size_t limit,
372 #if ZEND_DEBUG
373 const char *filename,
374 uint32_t lineno,
375 #endif
376 size_t size)
377 {
378
379 heap->overflow = 1;
380 zend_try {
381 zend_error_noreturn(E_ERROR,
382 format,
383 limit,
384 #if ZEND_DEBUG
385 filename,
386 lineno,
387 #endif
388 size);
389 } zend_catch {
390 } zend_end_try();
391 heap->overflow = 0;
392 zend_bailout();
393 exit(1);
394 }
395
396 #ifdef _WIN32
397 void
stderr_last_error(char * msg)398 stderr_last_error(char *msg)
399 {
400 DWORD err = GetLastError();
401 char *buf = php_win32_error_to_msg(err);
402
403 if (!buf[0]) {
404 fprintf(stderr, "\n%s: [0x%08lx]\n", msg, err);
405 }
406 else {
407 fprintf(stderr, "\n%s: [0x%08lx] %s\n", msg, err, buf);
408 }
409
410 php_win32_error_msg_free(buf);
411 }
412 #endif
413
414 /*****************/
415 /* OS Allocation */
416 /*****************/
417
418 #ifndef HAVE_MREMAP
zend_mm_mmap_fixed(void * addr,size_t size)419 static void *zend_mm_mmap_fixed(void *addr, size_t size)
420 {
421 #ifdef _WIN32
422 return VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
423 #else
424 int flags = MAP_PRIVATE | MAP_ANON;
425 #if defined(MAP_EXCL)
426 flags |= MAP_FIXED | MAP_EXCL;
427 #endif
428 /* MAP_FIXED leads to discarding of the old mapping, so it can't be used. */
429 void *ptr = mmap(addr, size, PROT_READ | PROT_WRITE, flags /*| MAP_POPULATE | MAP_HUGETLB*/, ZEND_MM_FD, 0);
430
431 if (ptr == MAP_FAILED) {
432 #if ZEND_MM_ERROR && !defined(MAP_EXCL)
433 fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno));
434 #endif
435 return NULL;
436 } else if (ptr != addr) {
437 if (munmap(ptr, size) != 0) {
438 #if ZEND_MM_ERROR
439 fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno));
440 #endif
441 }
442 return NULL;
443 }
444 return ptr;
445 #endif
446 }
447 #endif
448
zend_mm_mmap(size_t size)449 static void *zend_mm_mmap(size_t size)
450 {
451 #ifdef _WIN32
452 void *ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
453
454 if (ptr == NULL) {
455 #if ZEND_MM_ERROR
456 stderr_last_error("VirtualAlloc() failed");
457 #endif
458 return NULL;
459 }
460 return ptr;
461 #else
462 void *ptr;
463
464 #ifdef MAP_HUGETLB
465 if (zend_mm_use_huge_pages && size == ZEND_MM_CHUNK_SIZE) {
466 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_HUGETLB, -1, 0);
467 if (ptr != MAP_FAILED) {
468 return ptr;
469 }
470 }
471 #endif
472
473 ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, ZEND_MM_FD, 0);
474
475 if (ptr == MAP_FAILED) {
476 #if ZEND_MM_ERROR
477 fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno));
478 #endif
479 return NULL;
480 }
481 return ptr;
482 #endif
483 }
484
zend_mm_munmap(void * addr,size_t size)485 static void zend_mm_munmap(void *addr, size_t size)
486 {
487 #ifdef _WIN32
488 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
489 #if ZEND_MM_ERROR
490 stderr_last_error("VirtualFree() failed");
491 #endif
492 }
493 #else
494 if (munmap(addr, size) != 0) {
495 #if ZEND_MM_ERROR
496 fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno));
497 #endif
498 }
499 #endif
500 }
501
502 /***********/
503 /* Bitmask */
504 /***********/
505
506 /* number of trailing set (1) bits */
zend_mm_bitset_nts(zend_mm_bitset bitset)507 static zend_always_inline int zend_mm_bitset_nts(zend_mm_bitset bitset)
508 {
509 #if (defined(__GNUC__) || __has_builtin(__builtin_ctzl)) && SIZEOF_ZEND_LONG == SIZEOF_LONG && defined(PHP_HAVE_BUILTIN_CTZL)
510 return __builtin_ctzl(~bitset);
511 #elif (defined(__GNUC__) || __has_builtin(__builtin_ctzll)) && defined(PHP_HAVE_BUILTIN_CTZLL)
512 return __builtin_ctzll(~bitset);
513 #elif defined(_WIN32)
514 unsigned long index;
515
516 #if defined(_WIN64)
517 if (!BitScanForward64(&index, ~bitset)) {
518 #else
519 if (!BitScanForward(&index, ~bitset)) {
520 #endif
521 /* undefined behavior */
522 return 32;
523 }
524
525 return (int)index;
526 #else
527 int n;
528
529 if (bitset == (zend_mm_bitset)-1) return ZEND_MM_BITSET_LEN;
530
531 n = 0;
532 #if SIZEOF_ZEND_LONG == 8
533 if (sizeof(zend_mm_bitset) == 8) {
534 if ((bitset & 0xffffffff) == 0xffffffff) {n += 32; bitset = bitset >> Z_UL(32);}
535 }
536 #endif
537 if ((bitset & 0x0000ffff) == 0x0000ffff) {n += 16; bitset = bitset >> 16;}
538 if ((bitset & 0x000000ff) == 0x000000ff) {n += 8; bitset = bitset >> 8;}
539 if ((bitset & 0x0000000f) == 0x0000000f) {n += 4; bitset = bitset >> 4;}
540 if ((bitset & 0x00000003) == 0x00000003) {n += 2; bitset = bitset >> 2;}
541 return n + (bitset & 1);
542 #endif
543 }
544
545 static zend_always_inline int zend_mm_bitset_is_set(zend_mm_bitset *bitset, int bit)
546 {
547 return ZEND_BIT_TEST(bitset, bit);
548 }
549
550 static zend_always_inline void zend_mm_bitset_set_bit(zend_mm_bitset *bitset, int bit)
551 {
552 bitset[bit / ZEND_MM_BITSET_LEN] |= (Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
553 }
554
555 static zend_always_inline void zend_mm_bitset_reset_bit(zend_mm_bitset *bitset, int bit)
556 {
557 bitset[bit / ZEND_MM_BITSET_LEN] &= ~(Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1)));
558 }
559
560 static zend_always_inline void zend_mm_bitset_set_range(zend_mm_bitset *bitset, int start, int len)
561 {
562 if (len == 1) {
563 zend_mm_bitset_set_bit(bitset, start);
564 } else {
565 int pos = start / ZEND_MM_BITSET_LEN;
566 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
567 int bit = start & (ZEND_MM_BITSET_LEN - 1);
568 zend_mm_bitset tmp;
569
570 if (pos != end) {
571 /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
572 tmp = (zend_mm_bitset)-1 << bit;
573 bitset[pos++] |= tmp;
574 while (pos != end) {
575 /* set all bits */
576 bitset[pos++] = (zend_mm_bitset)-1;
577 }
578 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
579 /* set bits from "0" to "end" */
580 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
581 bitset[pos] |= tmp;
582 } else {
583 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
584 /* set bits from "bit" to "end" */
585 tmp = (zend_mm_bitset)-1 << bit;
586 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
587 bitset[pos] |= tmp;
588 }
589 }
590 }
591
592 static zend_always_inline void zend_mm_bitset_reset_range(zend_mm_bitset *bitset, int start, int len)
593 {
594 if (len == 1) {
595 zend_mm_bitset_reset_bit(bitset, start);
596 } else {
597 int pos = start / ZEND_MM_BITSET_LEN;
598 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
599 int bit = start & (ZEND_MM_BITSET_LEN - 1);
600 zend_mm_bitset tmp;
601
602 if (pos != end) {
603 /* reset bits from "bit" to ZEND_MM_BITSET_LEN-1 */
604 tmp = ~((Z_UL(1) << bit) - 1);
605 bitset[pos++] &= ~tmp;
606 while (pos != end) {
607 /* set all bits */
608 bitset[pos++] = 0;
609 }
610 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
611 /* reset bits from "0" to "end" */
612 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
613 bitset[pos] &= ~tmp;
614 } else {
615 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
616 /* reset bits from "bit" to "end" */
617 tmp = (zend_mm_bitset)-1 << bit;
618 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
619 bitset[pos] &= ~tmp;
620 }
621 }
622 }
623
624 static zend_always_inline int zend_mm_bitset_is_free_range(zend_mm_bitset *bitset, int start, int len)
625 {
626 if (len == 1) {
627 return !zend_mm_bitset_is_set(bitset, start);
628 } else {
629 int pos = start / ZEND_MM_BITSET_LEN;
630 int end = (start + len - 1) / ZEND_MM_BITSET_LEN;
631 int bit = start & (ZEND_MM_BITSET_LEN - 1);
632 zend_mm_bitset tmp;
633
634 if (pos != end) {
635 /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */
636 tmp = (zend_mm_bitset)-1 << bit;
637 if ((bitset[pos++] & tmp) != 0) {
638 return 0;
639 }
640 while (pos != end) {
641 /* set all bits */
642 if (bitset[pos++] != 0) {
643 return 0;
644 }
645 }
646 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
647 /* set bits from "0" to "end" */
648 tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
649 return (bitset[pos] & tmp) == 0;
650 } else {
651 end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1);
652 /* set bits from "bit" to "end" */
653 tmp = (zend_mm_bitset)-1 << bit;
654 tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end);
655 return (bitset[pos] & tmp) == 0;
656 }
657 }
658 }
659
660 /**********/
661 /* Chunks */
662 /**********/
663
664 static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment)
665 {
666 void *ptr = zend_mm_mmap(size);
667
668 if (ptr == NULL) {
669 return NULL;
670 } else if (ZEND_MM_ALIGNED_OFFSET(ptr, alignment) == 0) {
671 #ifdef MADV_HUGEPAGE
672 if (zend_mm_use_huge_pages) {
673 madvise(ptr, size, MADV_HUGEPAGE);
674 }
675 #endif
676 return ptr;
677 } else {
678 size_t offset;
679
680 /* chunk has to be aligned */
681 zend_mm_munmap(ptr, size);
682 ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE);
683 #ifdef _WIN32
684 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
685 zend_mm_munmap(ptr, size + alignment - REAL_PAGE_SIZE);
686 ptr = zend_mm_mmap_fixed((void*)((char*)ptr + (alignment - offset)), size);
687 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
688 if (offset != 0) {
689 zend_mm_munmap(ptr, size);
690 return NULL;
691 }
692 return ptr;
693 #else
694 offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment);
695 if (offset != 0) {
696 offset = alignment - offset;
697 zend_mm_munmap(ptr, offset);
698 ptr = (char*)ptr + offset;
699 alignment -= offset;
700 }
701 if (alignment > REAL_PAGE_SIZE) {
702 zend_mm_munmap((char*)ptr + size, alignment - REAL_PAGE_SIZE);
703 }
704 # ifdef MADV_HUGEPAGE
705 if (zend_mm_use_huge_pages) {
706 madvise(ptr, size, MADV_HUGEPAGE);
707 }
708 # endif
709 #endif
710 return ptr;
711 }
712 }
713
714 static void *zend_mm_chunk_alloc(zend_mm_heap *heap, size_t size, size_t alignment)
715 {
716 #if ZEND_MM_STORAGE
717 if (UNEXPECTED(heap->storage)) {
718 void *ptr = heap->storage->handlers.chunk_alloc(heap->storage, size, alignment);
719 ZEND_ASSERT(((zend_uintptr_t)((char*)ptr + (alignment-1)) & (alignment-1)) == (zend_uintptr_t)ptr);
720 return ptr;
721 }
722 #endif
723 return zend_mm_chunk_alloc_int(size, alignment);
724 }
725
726 static void zend_mm_chunk_free(zend_mm_heap *heap, void *addr, size_t size)
727 {
728 #if ZEND_MM_STORAGE
729 if (UNEXPECTED(heap->storage)) {
730 heap->storage->handlers.chunk_free(heap->storage, addr, size);
731 return;
732 }
733 #endif
734 zend_mm_munmap(addr, size);
735 }
736
737 static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
738 {
739 #if ZEND_MM_STORAGE
740 if (UNEXPECTED(heap->storage)) {
741 if (heap->storage->handlers.chunk_truncate) {
742 return heap->storage->handlers.chunk_truncate(heap->storage, addr, old_size, new_size);
743 } else {
744 return 0;
745 }
746 }
747 #endif
748 #ifndef _WIN32
749 zend_mm_munmap((char*)addr + new_size, old_size - new_size);
750 return 1;
751 #else
752 return 0;
753 #endif
754 }
755
756 static int zend_mm_chunk_extend(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size)
757 {
758 #if ZEND_MM_STORAGE
759 if (UNEXPECTED(heap->storage)) {
760 if (heap->storage->handlers.chunk_extend) {
761 return heap->storage->handlers.chunk_extend(heap->storage, addr, old_size, new_size);
762 } else {
763 return 0;
764 }
765 }
766 #endif
767 #ifdef HAVE_MREMAP
768 /* We don't use MREMAP_MAYMOVE due to alignment requirements. */
769 void *ptr = mremap(addr, old_size, new_size, 0);
770 if (ptr == MAP_FAILED) {
771 return 0;
772 }
773 /* Sanity check: The mapping shouldn't have moved. */
774 ZEND_ASSERT(ptr == addr);
775 return 1;
776 #elif !defined(_WIN32)
777 return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL);
778 #else
779 return 0;
780 #endif
781 }
782
783 static zend_always_inline void zend_mm_chunk_init(zend_mm_heap *heap, zend_mm_chunk *chunk)
784 {
785 chunk->heap = heap;
786 chunk->next = heap->main_chunk;
787 chunk->prev = heap->main_chunk->prev;
788 chunk->prev->next = chunk;
789 chunk->next->prev = chunk;
790 /* mark first pages as allocated */
791 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
792 chunk->free_tail = ZEND_MM_FIRST_PAGE;
793 /* the younger chunks have bigger number */
794 chunk->num = chunk->prev->num + 1;
795 /* mark first pages as allocated */
796 chunk->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
797 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
798 }
799
800 /***********************/
801 /* Huge Runs (forward) */
802 /***********************/
803
804 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
805 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
806 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
807
808 #if ZEND_DEBUG
809 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
810 #else
811 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC);
812 #endif
813
814 /**************/
815 /* Large Runs */
816 /**************/
817
818 #if ZEND_DEBUG
819 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
820 #else
821 static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
822 #endif
823 {
824 zend_mm_chunk *chunk = heap->main_chunk;
825 uint32_t page_num, len;
826 int steps = 0;
827
828 while (1) {
829 if (UNEXPECTED(chunk->free_pages < pages_count)) {
830 goto not_found;
831 #if 0
832 } else if (UNEXPECTED(chunk->free_pages + chunk->free_tail == ZEND_MM_PAGES)) {
833 if (UNEXPECTED(ZEND_MM_PAGES - chunk->free_tail < pages_count)) {
834 goto not_found;
835 } else {
836 page_num = chunk->free_tail;
837 goto found;
838 }
839 } else if (0) {
840 /* First-Fit Search */
841 int free_tail = chunk->free_tail;
842 zend_mm_bitset *bitset = chunk->free_map;
843 zend_mm_bitset tmp = *(bitset++);
844 int i = 0;
845
846 while (1) {
847 /* skip allocated blocks */
848 while (tmp == (zend_mm_bitset)-1) {
849 i += ZEND_MM_BITSET_LEN;
850 if (i == ZEND_MM_PAGES) {
851 goto not_found;
852 }
853 tmp = *(bitset++);
854 }
855 /* find first 0 bit */
856 page_num = i + zend_mm_bitset_nts(tmp);
857 /* reset bits from 0 to "bit" */
858 tmp &= tmp + 1;
859 /* skip free blocks */
860 while (tmp == 0) {
861 i += ZEND_MM_BITSET_LEN;
862 len = i - page_num;
863 if (len >= pages_count) {
864 goto found;
865 } else if (i >= free_tail) {
866 goto not_found;
867 }
868 tmp = *(bitset++);
869 }
870 /* find first 1 bit */
871 len = (i + zend_ulong_ntz(tmp)) - page_num;
872 if (len >= pages_count) {
873 goto found;
874 }
875 /* set bits from 0 to "bit" */
876 tmp |= tmp - 1;
877 }
878 #endif
879 } else {
880 /* Best-Fit Search */
881 int best = -1;
882 uint32_t best_len = ZEND_MM_PAGES;
883 uint32_t free_tail = chunk->free_tail;
884 zend_mm_bitset *bitset = chunk->free_map;
885 zend_mm_bitset tmp = *(bitset++);
886 uint32_t i = 0;
887
888 while (1) {
889 /* skip allocated blocks */
890 while (tmp == (zend_mm_bitset)-1) {
891 i += ZEND_MM_BITSET_LEN;
892 if (i == ZEND_MM_PAGES) {
893 if (best > 0) {
894 page_num = best;
895 goto found;
896 } else {
897 goto not_found;
898 }
899 }
900 tmp = *(bitset++);
901 }
902 /* find first 0 bit */
903 page_num = i + zend_mm_bitset_nts(tmp);
904 /* reset bits from 0 to "bit" */
905 tmp &= tmp + 1;
906 /* skip free blocks */
907 while (tmp == 0) {
908 i += ZEND_MM_BITSET_LEN;
909 if (i >= free_tail || i == ZEND_MM_PAGES) {
910 len = ZEND_MM_PAGES - page_num;
911 if (len >= pages_count && len < best_len) {
912 chunk->free_tail = page_num + pages_count;
913 goto found;
914 } else {
915 /* set accurate value */
916 chunk->free_tail = page_num;
917 if (best > 0) {
918 page_num = best;
919 goto found;
920 } else {
921 goto not_found;
922 }
923 }
924 }
925 tmp = *(bitset++);
926 }
927 /* find first 1 bit */
928 len = i + zend_ulong_ntz(tmp) - page_num;
929 if (len >= pages_count) {
930 if (len == pages_count) {
931 goto found;
932 } else if (len < best_len) {
933 best_len = len;
934 best = page_num;
935 }
936 }
937 /* set bits from 0 to "bit" */
938 tmp |= tmp - 1;
939 }
940 }
941
942 not_found:
943 if (chunk->next == heap->main_chunk) {
944 get_chunk:
945 if (heap->cached_chunks) {
946 heap->cached_chunks_count--;
947 chunk = heap->cached_chunks;
948 heap->cached_chunks = chunk->next;
949 } else {
950 #if ZEND_MM_LIMIT
951 if (UNEXPECTED(ZEND_MM_CHUNK_SIZE > heap->limit - heap->real_size)) {
952 if (zend_mm_gc(heap)) {
953 goto get_chunk;
954 } else if (heap->overflow == 0) {
955 #if ZEND_DEBUG
956 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
957 #else
958 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, ZEND_MM_PAGE_SIZE * pages_count);
959 #endif
960 return NULL;
961 }
962 }
963 #endif
964 chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
965 if (UNEXPECTED(chunk == NULL)) {
966 /* insufficient memory */
967 if (zend_mm_gc(heap) &&
968 (chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE)) != NULL) {
969 /* pass */
970 } else {
971 #if !ZEND_MM_LIMIT
972 zend_mm_safe_error(heap, "Out of memory");
973 #elif ZEND_DEBUG
974 zend_mm_safe_error(heap, "Out of memory (allocated %zu) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
975 #else
976 zend_mm_safe_error(heap, "Out of memory (allocated %zu) (tried to allocate %zu bytes)", heap->real_size, ZEND_MM_PAGE_SIZE * pages_count);
977 #endif
978 return NULL;
979 }
980 }
981 #if ZEND_MM_STAT
982 do {
983 size_t size = heap->real_size + ZEND_MM_CHUNK_SIZE;
984 size_t peak = MAX(heap->real_peak, size);
985 heap->real_size = size;
986 heap->real_peak = peak;
987 } while (0);
988 #elif ZEND_MM_LIMIT
989 heap->real_size += ZEND_MM_CHUNK_SIZE;
990
991 #endif
992 }
993 heap->chunks_count++;
994 if (heap->chunks_count > heap->peak_chunks_count) {
995 heap->peak_chunks_count = heap->chunks_count;
996 }
997 zend_mm_chunk_init(heap, chunk);
998 page_num = ZEND_MM_FIRST_PAGE;
999 len = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1000 goto found;
1001 } else {
1002 chunk = chunk->next;
1003 steps++;
1004 }
1005 }
1006
1007 found:
1008 if (steps > 2 && pages_count < 8) {
1009 /* move chunk into the head of the linked-list */
1010 chunk->prev->next = chunk->next;
1011 chunk->next->prev = chunk->prev;
1012 chunk->next = heap->main_chunk->next;
1013 chunk->prev = heap->main_chunk;
1014 chunk->prev->next = chunk;
1015 chunk->next->prev = chunk;
1016 }
1017 /* mark run as allocated */
1018 chunk->free_pages -= pages_count;
1019 zend_mm_bitset_set_range(chunk->free_map, page_num, pages_count);
1020 chunk->map[page_num] = ZEND_MM_LRUN(pages_count);
1021 if (page_num == chunk->free_tail) {
1022 chunk->free_tail = page_num + pages_count;
1023 }
1024 return ZEND_MM_PAGE_ADDR(chunk, page_num);
1025 }
1026
1027 static zend_always_inline void *zend_mm_alloc_large_ex(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1028 {
1029 int pages_count = (int)ZEND_MM_SIZE_TO_NUM(size, ZEND_MM_PAGE_SIZE);
1030 #if ZEND_DEBUG
1031 void *ptr = zend_mm_alloc_pages(heap, pages_count, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1032 #else
1033 void *ptr = zend_mm_alloc_pages(heap, pages_count ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1034 #endif
1035 #if ZEND_MM_STAT
1036 do {
1037 size_t size = heap->size + pages_count * ZEND_MM_PAGE_SIZE;
1038 size_t peak = MAX(heap->peak, size);
1039 heap->size = size;
1040 heap->peak = peak;
1041 } while (0);
1042 #endif
1043 return ptr;
1044 }
1045
1046 #if ZEND_DEBUG
1047 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1048 {
1049 return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1050 }
1051 #else
1052 static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1053 {
1054 return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1055 }
1056 #endif
1057
1058 static zend_always_inline void zend_mm_delete_chunk(zend_mm_heap *heap, zend_mm_chunk *chunk)
1059 {
1060 chunk->next->prev = chunk->prev;
1061 chunk->prev->next = chunk->next;
1062 heap->chunks_count--;
1063 if (heap->chunks_count + heap->cached_chunks_count < heap->avg_chunks_count + 0.1
1064 || (heap->chunks_count == heap->last_chunks_delete_boundary
1065 && heap->last_chunks_delete_count >= 4)) {
1066 /* delay deletion */
1067 heap->cached_chunks_count++;
1068 chunk->next = heap->cached_chunks;
1069 heap->cached_chunks = chunk;
1070 } else {
1071 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1072 heap->real_size -= ZEND_MM_CHUNK_SIZE;
1073 #endif
1074 if (!heap->cached_chunks) {
1075 if (heap->chunks_count != heap->last_chunks_delete_boundary) {
1076 heap->last_chunks_delete_boundary = heap->chunks_count;
1077 heap->last_chunks_delete_count = 0;
1078 } else {
1079 heap->last_chunks_delete_count++;
1080 }
1081 }
1082 if (!heap->cached_chunks || chunk->num > heap->cached_chunks->num) {
1083 zend_mm_chunk_free(heap, chunk, ZEND_MM_CHUNK_SIZE);
1084 } else {
1085 //TODO: select the best chunk to delete???
1086 chunk->next = heap->cached_chunks->next;
1087 zend_mm_chunk_free(heap, heap->cached_chunks, ZEND_MM_CHUNK_SIZE);
1088 heap->cached_chunks = chunk;
1089 }
1090 }
1091 }
1092
1093 static zend_always_inline void zend_mm_free_pages_ex(zend_mm_heap *heap, zend_mm_chunk *chunk, uint32_t page_num, uint32_t pages_count, int free_chunk)
1094 {
1095 chunk->free_pages += pages_count;
1096 zend_mm_bitset_reset_range(chunk->free_map, page_num, pages_count);
1097 chunk->map[page_num] = 0;
1098 if (chunk->free_tail == page_num + pages_count) {
1099 /* this setting may be not accurate */
1100 chunk->free_tail = page_num;
1101 }
1102 if (free_chunk && chunk != heap->main_chunk && chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
1103 zend_mm_delete_chunk(heap, chunk);
1104 }
1105 }
1106
1107 static zend_never_inline void zend_mm_free_pages(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1108 {
1109 zend_mm_free_pages_ex(heap, chunk, page_num, pages_count, 1);
1110 }
1111
1112 static zend_always_inline void zend_mm_free_large(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count)
1113 {
1114 #if ZEND_MM_STAT
1115 heap->size -= pages_count * ZEND_MM_PAGE_SIZE;
1116 #endif
1117 zend_mm_free_pages(heap, chunk, page_num, pages_count);
1118 }
1119
1120 /**************/
1121 /* Small Runs */
1122 /**************/
1123
1124 /* higher set bit number (0->N/A, 1->1, 2->2, 4->3, 8->4, 127->7, 128->8 etc) */
1125 static zend_always_inline int zend_mm_small_size_to_bit(int size)
1126 {
1127 #if (defined(__GNUC__) || __has_builtin(__builtin_clz)) && defined(PHP_HAVE_BUILTIN_CLZ)
1128 return (__builtin_clz(size) ^ 0x1f) + 1;
1129 #elif defined(_WIN32)
1130 unsigned long index;
1131
1132 if (!BitScanReverse(&index, (unsigned long)size)) {
1133 /* undefined behavior */
1134 return 64;
1135 }
1136
1137 return (((31 - (int)index) ^ 0x1f) + 1);
1138 #else
1139 int n = 16;
1140 if (size <= 0x00ff) {n -= 8; size = size << 8;}
1141 if (size <= 0x0fff) {n -= 4; size = size << 4;}
1142 if (size <= 0x3fff) {n -= 2; size = size << 2;}
1143 if (size <= 0x7fff) {n -= 1;}
1144 return n;
1145 #endif
1146 }
1147
1148 #ifndef MAX
1149 # define MAX(a, b) (((a) > (b)) ? (a) : (b))
1150 #endif
1151
1152 #ifndef MIN
1153 # define MIN(a, b) (((a) < (b)) ? (a) : (b))
1154 #endif
1155
1156 static zend_always_inline int zend_mm_small_size_to_bin(size_t size)
1157 {
1158 #if 0
1159 int n;
1160 /*0, 1, 2, 3, 4, 5, 6, 7, 8, 9 10, 11, 12*/
1161 static const int f1[] = { 3, 3, 3, 3, 3, 3, 3, 4, 5, 6, 7, 8, 9};
1162 static const int f2[] = { 0, 0, 0, 0, 0, 0, 0, 4, 8, 12, 16, 20, 24};
1163
1164 if (UNEXPECTED(size <= 2)) return 0;
1165 n = zend_mm_small_size_to_bit(size - 1);
1166 return ((size-1) >> f1[n]) + f2[n];
1167 #else
1168 unsigned int t1, t2;
1169
1170 if (size <= 64) {
1171 /* we need to support size == 0 ... */
1172 return (size - !!size) >> 3;
1173 } else {
1174 t1 = size - 1;
1175 t2 = zend_mm_small_size_to_bit(t1) - 3;
1176 t1 = t1 >> t2;
1177 t2 = t2 - 3;
1178 t2 = t2 << 2;
1179 return (int)(t1 + t2);
1180 }
1181 #endif
1182 }
1183
1184 #define ZEND_MM_SMALL_SIZE_TO_BIN(size) zend_mm_small_size_to_bin(size)
1185
1186 static zend_never_inline void *zend_mm_alloc_small_slow(zend_mm_heap *heap, uint32_t bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1187 {
1188 zend_mm_chunk *chunk;
1189 int page_num;
1190 zend_mm_bin *bin;
1191 zend_mm_free_slot *p, *end;
1192
1193 #if ZEND_DEBUG
1194 bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num], bin_data_size[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1195 #else
1196 bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1197 #endif
1198 if (UNEXPECTED(bin == NULL)) {
1199 /* insufficient memory */
1200 return NULL;
1201 }
1202
1203 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(bin, ZEND_MM_CHUNK_SIZE);
1204 page_num = ZEND_MM_ALIGNED_OFFSET(bin, ZEND_MM_CHUNK_SIZE) / ZEND_MM_PAGE_SIZE;
1205 chunk->map[page_num] = ZEND_MM_SRUN(bin_num);
1206 if (bin_pages[bin_num] > 1) {
1207 uint32_t i = 1;
1208
1209 do {
1210 chunk->map[page_num+i] = ZEND_MM_NRUN(bin_num, i);
1211 i++;
1212 } while (i < bin_pages[bin_num]);
1213 }
1214
1215 /* create a linked list of elements from 1 to last */
1216 end = (zend_mm_free_slot*)((char*)bin + (bin_data_size[bin_num] * (bin_elements[bin_num] - 1)));
1217 heap->free_slot[bin_num] = p = (zend_mm_free_slot*)((char*)bin + bin_data_size[bin_num]);
1218 do {
1219 p->next_free_slot = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1220 #if ZEND_DEBUG
1221 do {
1222 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1223 dbg->size = 0;
1224 } while (0);
1225 #endif
1226 p = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]);
1227 } while (p != end);
1228
1229 /* terminate list using NULL */
1230 p->next_free_slot = NULL;
1231 #if ZEND_DEBUG
1232 do {
1233 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1234 dbg->size = 0;
1235 } while (0);
1236 #endif
1237
1238 /* return first element */
1239 return (char*)bin;
1240 }
1241
1242 static zend_always_inline void *zend_mm_alloc_small(zend_mm_heap *heap, int bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1243 {
1244 #if ZEND_MM_STAT
1245 do {
1246 size_t size = heap->size + bin_data_size[bin_num];
1247 size_t peak = MAX(heap->peak, size);
1248 heap->size = size;
1249 heap->peak = peak;
1250 } while (0);
1251 #endif
1252
1253 if (EXPECTED(heap->free_slot[bin_num] != NULL)) {
1254 zend_mm_free_slot *p = heap->free_slot[bin_num];
1255 heap->free_slot[bin_num] = p->next_free_slot;
1256 return (void*)p;
1257 } else {
1258 return zend_mm_alloc_small_slow(heap, bin_num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1259 }
1260 }
1261
1262 static zend_always_inline void zend_mm_free_small(zend_mm_heap *heap, void *ptr, int bin_num)
1263 {
1264 zend_mm_free_slot *p;
1265
1266 #if ZEND_MM_STAT
1267 heap->size -= bin_data_size[bin_num];
1268 #endif
1269
1270 #if ZEND_DEBUG
1271 do {
1272 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1273 dbg->size = 0;
1274 } while (0);
1275 #endif
1276
1277 p = (zend_mm_free_slot*)ptr;
1278 p->next_free_slot = heap->free_slot[bin_num];
1279 heap->free_slot[bin_num] = p;
1280 }
1281
1282 /********/
1283 /* Heap */
1284 /********/
1285
1286 #if ZEND_DEBUG
1287 static zend_always_inline zend_mm_debug_info *zend_mm_get_debug_info(zend_mm_heap *heap, void *ptr)
1288 {
1289 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1290 zend_mm_chunk *chunk;
1291 int page_num;
1292 zend_mm_page_info info;
1293
1294 ZEND_MM_CHECK(page_offset != 0, "zend_mm_heap corrupted");
1295 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1296 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1297 info = chunk->map[page_num];
1298 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1299 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1300 int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1301 return (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1302 } else /* if (info & ZEND_MM_IS_LRUN) */ {
1303 int pages_count = ZEND_MM_LRUN_PAGES(info);
1304
1305 return (zend_mm_debug_info*)((char*)ptr + ZEND_MM_PAGE_SIZE * pages_count - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1306 }
1307 }
1308 #endif
1309
1310 static zend_always_inline void *zend_mm_alloc_heap(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1311 {
1312 void *ptr;
1313 #if ZEND_DEBUG
1314 size_t real_size = size;
1315 zend_mm_debug_info *dbg;
1316
1317 /* special handling for zero-size allocation */
1318 size = MAX(size, 1);
1319 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1320 if (UNEXPECTED(size < real_size)) {
1321 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", ZEND_MM_ALIGNED_SIZE(real_size), ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
1322 return NULL;
1323 }
1324 #endif
1325 if (EXPECTED(size <= ZEND_MM_MAX_SMALL_SIZE)) {
1326 ptr = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1327 #if ZEND_DEBUG
1328 dbg = zend_mm_get_debug_info(heap, ptr);
1329 dbg->size = real_size;
1330 dbg->filename = __zend_filename;
1331 dbg->orig_filename = __zend_orig_filename;
1332 dbg->lineno = __zend_lineno;
1333 dbg->orig_lineno = __zend_orig_lineno;
1334 #endif
1335 return ptr;
1336 } else if (EXPECTED(size <= ZEND_MM_MAX_LARGE_SIZE)) {
1337 ptr = zend_mm_alloc_large(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1338 #if ZEND_DEBUG
1339 dbg = zend_mm_get_debug_info(heap, ptr);
1340 dbg->size = real_size;
1341 dbg->filename = __zend_filename;
1342 dbg->orig_filename = __zend_orig_filename;
1343 dbg->lineno = __zend_lineno;
1344 dbg->orig_lineno = __zend_orig_lineno;
1345 #endif
1346 return ptr;
1347 } else {
1348 #if ZEND_DEBUG
1349 size = real_size;
1350 #endif
1351 return zend_mm_alloc_huge(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1352 }
1353 }
1354
1355 static zend_always_inline void zend_mm_free_heap(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1356 {
1357 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1358
1359 if (UNEXPECTED(page_offset == 0)) {
1360 if (ptr != NULL) {
1361 zend_mm_free_huge(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1362 }
1363 } else {
1364 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1365 int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1366 zend_mm_page_info info = chunk->map[page_num];
1367
1368 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1369 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1370 zend_mm_free_small(heap, ptr, ZEND_MM_SRUN_BIN_NUM(info));
1371 } else /* if (info & ZEND_MM_IS_LRUN) */ {
1372 int pages_count = ZEND_MM_LRUN_PAGES(info);
1373
1374 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1375 zend_mm_free_large(heap, chunk, page_num, pages_count);
1376 }
1377 }
1378 }
1379
1380 static size_t zend_mm_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1381 {
1382 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1383
1384 if (UNEXPECTED(page_offset == 0)) {
1385 return zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1386 } else {
1387 zend_mm_chunk *chunk;
1388 #if 0 && ZEND_DEBUG
1389 zend_mm_debug_info *dbg = zend_mm_get_debug_info(heap, ptr);
1390 return dbg->size;
1391 #else
1392 int page_num;
1393 zend_mm_page_info info;
1394
1395 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1396 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1397 info = chunk->map[page_num];
1398 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1399 if (EXPECTED(info & ZEND_MM_IS_SRUN)) {
1400 return bin_data_size[ZEND_MM_SRUN_BIN_NUM(info)];
1401 } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1402 return ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1403 }
1404 #endif
1405 }
1406 }
1407
1408 static zend_never_inline void *zend_mm_realloc_slow(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1409 {
1410 void *ret;
1411
1412 #if ZEND_MM_STAT
1413 do {
1414 size_t orig_peak = heap->peak;
1415 #endif
1416 ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1417 memcpy(ret, ptr, copy_size);
1418 zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1419 #if ZEND_MM_STAT
1420 heap->peak = MAX(orig_peak, heap->size);
1421 } while (0);
1422 #endif
1423 return ret;
1424 }
1425
1426 static zend_never_inline void *zend_mm_realloc_huge(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1427 {
1428 size_t old_size;
1429 size_t new_size;
1430 #if ZEND_DEBUG
1431 size_t real_size;
1432 #endif
1433
1434 old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1435 #if ZEND_DEBUG
1436 real_size = size;
1437 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1438 #endif
1439 if (size > ZEND_MM_MAX_LARGE_SIZE) {
1440 #if ZEND_DEBUG
1441 size = real_size;
1442 #endif
1443 #ifdef ZEND_WIN32
1444 /* On Windows we don't have ability to extend huge blocks in-place.
1445 * We allocate them with 2MB size granularity, to avoid many
1446 * reallocations when they are extended by small pieces
1447 */
1448 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE));
1449 #else
1450 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE);
1451 #endif
1452 if (new_size == old_size) {
1453 #if ZEND_DEBUG
1454 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1455 #else
1456 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1457 #endif
1458 return ptr;
1459 } else if (new_size < old_size) {
1460 /* unmup tail */
1461 if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) {
1462 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1463 heap->real_size -= old_size - new_size;
1464 #endif
1465 #if ZEND_MM_STAT
1466 heap->size -= old_size - new_size;
1467 #endif
1468 #if ZEND_DEBUG
1469 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1470 #else
1471 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1472 #endif
1473 return ptr;
1474 }
1475 } else /* if (new_size > old_size) */ {
1476 #if ZEND_MM_LIMIT
1477 if (UNEXPECTED(new_size - old_size > heap->limit - heap->real_size)) {
1478 if (zend_mm_gc(heap) && new_size - old_size <= heap->limit - heap->real_size) {
1479 /* pass */
1480 } else if (heap->overflow == 0) {
1481 #if ZEND_DEBUG
1482 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1483 #else
1484 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1485 #endif
1486 return NULL;
1487 }
1488 }
1489 #endif
1490 /* try to map tail right after this block */
1491 if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) {
1492 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1493 heap->real_size += new_size - old_size;
1494 #endif
1495 #if ZEND_MM_STAT
1496 heap->real_peak = MAX(heap->real_peak, heap->real_size);
1497 heap->size += new_size - old_size;
1498 heap->peak = MAX(heap->peak, heap->size);
1499 #endif
1500 #if ZEND_DEBUG
1501 zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1502 #else
1503 zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1504 #endif
1505 return ptr;
1506 }
1507 }
1508 }
1509
1510 return zend_mm_realloc_slow(heap, ptr, size, MIN(old_size, copy_size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1511 }
1512
1513 static zend_always_inline void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, zend_bool use_copy_size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1514 {
1515 size_t page_offset;
1516 size_t old_size;
1517 size_t new_size;
1518 void *ret;
1519 #if ZEND_DEBUG
1520 zend_mm_debug_info *dbg;
1521 #endif
1522
1523 page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
1524 if (UNEXPECTED(page_offset == 0)) {
1525 if (EXPECTED(ptr == NULL)) {
1526 return _zend_mm_alloc(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1527 } else {
1528 return zend_mm_realloc_huge(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1529 }
1530 } else {
1531 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
1532 int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1533 zend_mm_page_info info = chunk->map[page_num];
1534 #if ZEND_DEBUG
1535 size_t real_size = size;
1536
1537 size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info));
1538 #endif
1539
1540 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1541 if (info & ZEND_MM_IS_SRUN) {
1542 int old_bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1543
1544 do {
1545 old_size = bin_data_size[old_bin_num];
1546
1547 /* Check if requested size fits into current bin */
1548 if (size <= old_size) {
1549 /* Check if truncation is necessary */
1550 if (old_bin_num > 0 && size < bin_data_size[old_bin_num - 1]) {
1551 /* truncation */
1552 ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1553 copy_size = use_copy_size ? MIN(size, copy_size) : size;
1554 memcpy(ret, ptr, copy_size);
1555 zend_mm_free_small(heap, ptr, old_bin_num);
1556 } else {
1557 /* reallocation in-place */
1558 ret = ptr;
1559 }
1560 } else if (size <= ZEND_MM_MAX_SMALL_SIZE) {
1561 /* small extension */
1562
1563 #if ZEND_MM_STAT
1564 do {
1565 size_t orig_peak = heap->peak;
1566 #endif
1567 ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1568 copy_size = use_copy_size ? MIN(old_size, copy_size) : old_size;
1569 memcpy(ret, ptr, copy_size);
1570 zend_mm_free_small(heap, ptr, old_bin_num);
1571 #if ZEND_MM_STAT
1572 heap->peak = MAX(orig_peak, heap->size);
1573 } while (0);
1574 #endif
1575 } else {
1576 /* slow reallocation */
1577 break;
1578 }
1579
1580 #if ZEND_DEBUG
1581 dbg = zend_mm_get_debug_info(heap, ret);
1582 dbg->size = real_size;
1583 dbg->filename = __zend_filename;
1584 dbg->orig_filename = __zend_orig_filename;
1585 dbg->lineno = __zend_lineno;
1586 dbg->orig_lineno = __zend_orig_lineno;
1587 #endif
1588 return ret;
1589 } while (0);
1590
1591 } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ {
1592 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
1593 old_size = ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE;
1594 if (size > ZEND_MM_MAX_SMALL_SIZE && size <= ZEND_MM_MAX_LARGE_SIZE) {
1595 new_size = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE);
1596 if (new_size == old_size) {
1597 #if ZEND_DEBUG
1598 dbg = zend_mm_get_debug_info(heap, ptr);
1599 dbg->size = real_size;
1600 dbg->filename = __zend_filename;
1601 dbg->orig_filename = __zend_orig_filename;
1602 dbg->lineno = __zend_lineno;
1603 dbg->orig_lineno = __zend_orig_lineno;
1604 #endif
1605 return ptr;
1606 } else if (new_size < old_size) {
1607 /* free tail pages */
1608 int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1609 int rest_pages_count = (int)((old_size - new_size) / ZEND_MM_PAGE_SIZE);
1610
1611 #if ZEND_MM_STAT
1612 heap->size -= rest_pages_count * ZEND_MM_PAGE_SIZE;
1613 #endif
1614 chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1615 chunk->free_pages += rest_pages_count;
1616 zend_mm_bitset_reset_range(chunk->free_map, page_num + new_pages_count, rest_pages_count);
1617 #if ZEND_DEBUG
1618 dbg = zend_mm_get_debug_info(heap, ptr);
1619 dbg->size = real_size;
1620 dbg->filename = __zend_filename;
1621 dbg->orig_filename = __zend_orig_filename;
1622 dbg->lineno = __zend_lineno;
1623 dbg->orig_lineno = __zend_orig_lineno;
1624 #endif
1625 return ptr;
1626 } else /* if (new_size > old_size) */ {
1627 int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE);
1628 int old_pages_count = (int)(old_size / ZEND_MM_PAGE_SIZE);
1629
1630 /* try to allocate tail pages after this block */
1631 if (page_num + new_pages_count <= ZEND_MM_PAGES &&
1632 zend_mm_bitset_is_free_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count)) {
1633 #if ZEND_MM_STAT
1634 do {
1635 size_t size = heap->size + (new_size - old_size);
1636 size_t peak = MAX(heap->peak, size);
1637 heap->size = size;
1638 heap->peak = peak;
1639 } while (0);
1640 #endif
1641 chunk->free_pages -= new_pages_count - old_pages_count;
1642 zend_mm_bitset_set_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count);
1643 chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count);
1644 #if ZEND_DEBUG
1645 dbg = zend_mm_get_debug_info(heap, ptr);
1646 dbg->size = real_size;
1647 dbg->filename = __zend_filename;
1648 dbg->orig_filename = __zend_orig_filename;
1649 dbg->lineno = __zend_lineno;
1650 dbg->orig_lineno = __zend_orig_lineno;
1651 #endif
1652 return ptr;
1653 }
1654 }
1655 }
1656 }
1657 #if ZEND_DEBUG
1658 size = real_size;
1659 #endif
1660 }
1661
1662 copy_size = MIN(old_size, copy_size);
1663 return zend_mm_realloc_slow(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1664 }
1665
1666 /*********************/
1667 /* Huge Runs (again) */
1668 /*********************/
1669
1670 #if ZEND_DEBUG
1671 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1672 #else
1673 static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1674 #endif
1675 {
1676 zend_mm_huge_list *list = (zend_mm_huge_list*)zend_mm_alloc_heap(heap, sizeof(zend_mm_huge_list) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1677 list->ptr = ptr;
1678 list->size = size;
1679 list->next = heap->huge_list;
1680 #if ZEND_DEBUG
1681 list->dbg.size = dbg_size;
1682 list->dbg.filename = __zend_filename;
1683 list->dbg.orig_filename = __zend_orig_filename;
1684 list->dbg.lineno = __zend_lineno;
1685 list->dbg.orig_lineno = __zend_orig_lineno;
1686 #endif
1687 heap->huge_list = list;
1688 }
1689
1690 static size_t zend_mm_del_huge_block(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1691 {
1692 zend_mm_huge_list *prev = NULL;
1693 zend_mm_huge_list *list = heap->huge_list;
1694 while (list != NULL) {
1695 if (list->ptr == ptr) {
1696 size_t size;
1697
1698 if (prev) {
1699 prev->next = list->next;
1700 } else {
1701 heap->huge_list = list->next;
1702 }
1703 size = list->size;
1704 zend_mm_free_heap(heap, list ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1705 return size;
1706 }
1707 prev = list;
1708 list = list->next;
1709 }
1710 ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1711 return 0;
1712 }
1713
1714 static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1715 {
1716 zend_mm_huge_list *list = heap->huge_list;
1717 while (list != NULL) {
1718 if (list->ptr == ptr) {
1719 return list->size;
1720 }
1721 list = list->next;
1722 }
1723 ZEND_MM_CHECK(0, "zend_mm_heap corrupted");
1724 return 0;
1725 }
1726
1727 #if ZEND_DEBUG
1728 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1729 #else
1730 static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1731 #endif
1732 {
1733 zend_mm_huge_list *list = heap->huge_list;
1734 while (list != NULL) {
1735 if (list->ptr == ptr) {
1736 list->size = size;
1737 #if ZEND_DEBUG
1738 list->dbg.size = dbg_size;
1739 list->dbg.filename = __zend_filename;
1740 list->dbg.orig_filename = __zend_orig_filename;
1741 list->dbg.lineno = __zend_lineno;
1742 list->dbg.orig_lineno = __zend_orig_lineno;
1743 #endif
1744 return;
1745 }
1746 list = list->next;
1747 }
1748 }
1749
1750 static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1751 {
1752 #ifdef ZEND_WIN32
1753 /* On Windows we don't have ability to extend huge blocks in-place.
1754 * We allocate them with 2MB size granularity, to avoid many
1755 * reallocations when they are extended by small pieces
1756 */
1757 size_t alignment = MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE);
1758 #else
1759 size_t alignment = REAL_PAGE_SIZE;
1760 #endif
1761 size_t new_size = ZEND_MM_ALIGNED_SIZE_EX(size, alignment);
1762 void *ptr;
1763
1764 if (UNEXPECTED(new_size < size)) {
1765 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", size, alignment);
1766 }
1767
1768 #if ZEND_MM_LIMIT
1769 if (UNEXPECTED(new_size > heap->limit - heap->real_size)) {
1770 if (zend_mm_gc(heap) && new_size <= heap->limit - heap->real_size) {
1771 /* pass */
1772 } else if (heap->overflow == 0) {
1773 #if ZEND_DEBUG
1774 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size);
1775 #else
1776 zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size);
1777 #endif
1778 return NULL;
1779 }
1780 }
1781 #endif
1782 ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE);
1783 if (UNEXPECTED(ptr == NULL)) {
1784 /* insufficient memory */
1785 if (zend_mm_gc(heap) &&
1786 (ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE)) != NULL) {
1787 /* pass */
1788 } else {
1789 #if !ZEND_MM_LIMIT
1790 zend_mm_safe_error(heap, "Out of memory");
1791 #elif ZEND_DEBUG
1792 zend_mm_safe_error(heap, "Out of memory (allocated %zu) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size);
1793 #else
1794 zend_mm_safe_error(heap, "Out of memory (allocated %zu) (tried to allocate %zu bytes)", heap->real_size, size);
1795 #endif
1796 return NULL;
1797 }
1798 }
1799 #if ZEND_DEBUG
1800 zend_mm_add_huge_block(heap, ptr, new_size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1801 #else
1802 zend_mm_add_huge_block(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1803 #endif
1804 #if ZEND_MM_STAT
1805 do {
1806 size_t size = heap->real_size + new_size;
1807 size_t peak = MAX(heap->real_peak, size);
1808 heap->real_size = size;
1809 heap->real_peak = peak;
1810 } while (0);
1811 do {
1812 size_t size = heap->size + new_size;
1813 size_t peak = MAX(heap->peak, size);
1814 heap->size = size;
1815 heap->peak = peak;
1816 } while (0);
1817 #elif ZEND_MM_LIMIT
1818 heap->real_size += new_size;
1819 #endif
1820 return ptr;
1821 }
1822
1823 static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
1824 {
1825 size_t size;
1826
1827 ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE) == 0, "zend_mm_heap corrupted");
1828 size = zend_mm_del_huge_block(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
1829 zend_mm_chunk_free(heap, ptr, size);
1830 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1831 heap->real_size -= size;
1832 #endif
1833 #if ZEND_MM_STAT
1834 heap->size -= size;
1835 #endif
1836 }
1837
1838 /******************/
1839 /* Initialization */
1840 /******************/
1841
1842 static zend_mm_heap *zend_mm_init(void)
1843 {
1844 zend_mm_chunk *chunk = (zend_mm_chunk*)zend_mm_chunk_alloc_int(ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
1845 zend_mm_heap *heap;
1846
1847 if (UNEXPECTED(chunk == NULL)) {
1848 #if ZEND_MM_ERROR
1849 #ifdef _WIN32
1850 stderr_last_error("Can't initialize heap");
1851 #else
1852 fprintf(stderr, "\nCan't initialize heap: [%d] %s\n", errno, strerror(errno));
1853 #endif
1854 #endif
1855 return NULL;
1856 }
1857 heap = &chunk->heap_slot;
1858 chunk->heap = heap;
1859 chunk->next = chunk;
1860 chunk->prev = chunk;
1861 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
1862 chunk->free_tail = ZEND_MM_FIRST_PAGE;
1863 chunk->num = 0;
1864 chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
1865 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
1866 heap->main_chunk = chunk;
1867 heap->cached_chunks = NULL;
1868 heap->chunks_count = 1;
1869 heap->peak_chunks_count = 1;
1870 heap->cached_chunks_count = 0;
1871 heap->avg_chunks_count = 1.0;
1872 heap->last_chunks_delete_boundary = 0;
1873 heap->last_chunks_delete_count = 0;
1874 #if ZEND_MM_STAT || ZEND_MM_LIMIT
1875 heap->real_size = ZEND_MM_CHUNK_SIZE;
1876 #endif
1877 #if ZEND_MM_STAT
1878 heap->real_peak = ZEND_MM_CHUNK_SIZE;
1879 heap->size = 0;
1880 heap->peak = 0;
1881 #endif
1882 #if ZEND_MM_LIMIT
1883 heap->limit = ((size_t)Z_L(-1) >> (size_t)Z_L(1));
1884 heap->overflow = 0;
1885 #endif
1886 #if ZEND_MM_CUSTOM
1887 heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
1888 #endif
1889 #if ZEND_MM_STORAGE
1890 heap->storage = NULL;
1891 #endif
1892 heap->huge_list = NULL;
1893 return heap;
1894 }
1895
1896 ZEND_API size_t zend_mm_gc(zend_mm_heap *heap)
1897 {
1898 zend_mm_free_slot *p, **q;
1899 zend_mm_chunk *chunk;
1900 size_t page_offset;
1901 int page_num;
1902 zend_mm_page_info info;
1903 uint32_t i, free_counter;
1904 int has_free_pages;
1905 size_t collected = 0;
1906
1907 #if ZEND_MM_CUSTOM
1908 if (heap->use_custom_heap) {
1909 return 0;
1910 }
1911 #endif
1912
1913 for (i = 0; i < ZEND_MM_BINS; i++) {
1914 has_free_pages = 0;
1915 p = heap->free_slot[i];
1916 while (p != NULL) {
1917 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
1918 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1919 page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
1920 ZEND_ASSERT(page_offset != 0);
1921 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1922 info = chunk->map[page_num];
1923 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1924 if (info & ZEND_MM_IS_LRUN) {
1925 page_num -= ZEND_MM_NRUN_OFFSET(info);
1926 info = chunk->map[page_num];
1927 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1928 ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
1929 }
1930 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
1931 free_counter = ZEND_MM_SRUN_FREE_COUNTER(info) + 1;
1932 if (free_counter == bin_elements[i]) {
1933 has_free_pages = 1;
1934 }
1935 chunk->map[page_num] = ZEND_MM_SRUN_EX(i, free_counter);
1936 p = p->next_free_slot;
1937 }
1938
1939 if (!has_free_pages) {
1940 continue;
1941 }
1942
1943 q = &heap->free_slot[i];
1944 p = *q;
1945 while (p != NULL) {
1946 chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE);
1947 ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted");
1948 page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE);
1949 ZEND_ASSERT(page_offset != 0);
1950 page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE);
1951 info = chunk->map[page_num];
1952 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1953 if (info & ZEND_MM_IS_LRUN) {
1954 page_num -= ZEND_MM_NRUN_OFFSET(info);
1955 info = chunk->map[page_num];
1956 ZEND_ASSERT(info & ZEND_MM_IS_SRUN);
1957 ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN));
1958 }
1959 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i);
1960 if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[i]) {
1961 /* remove from cache */
1962 p = p->next_free_slot;
1963 *q = p;
1964 } else {
1965 q = &p->next_free_slot;
1966 p = *q;
1967 }
1968 }
1969 }
1970
1971 chunk = heap->main_chunk;
1972 do {
1973 i = ZEND_MM_FIRST_PAGE;
1974 while (i < chunk->free_tail) {
1975 if (zend_mm_bitset_is_set(chunk->free_map, i)) {
1976 info = chunk->map[i];
1977 if (info & ZEND_MM_IS_SRUN) {
1978 int bin_num = ZEND_MM_SRUN_BIN_NUM(info);
1979 int pages_count = bin_pages[bin_num];
1980
1981 if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[bin_num]) {
1982 /* all elements are free */
1983 zend_mm_free_pages_ex(heap, chunk, i, pages_count, 0);
1984 collected += pages_count;
1985 } else {
1986 /* reset counter */
1987 chunk->map[i] = ZEND_MM_SRUN(bin_num);
1988 }
1989 i += bin_pages[bin_num];
1990 } else /* if (info & ZEND_MM_IS_LRUN) */ {
1991 i += ZEND_MM_LRUN_PAGES(info);
1992 }
1993 } else {
1994 i++;
1995 }
1996 }
1997 if (chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) {
1998 zend_mm_chunk *next_chunk = chunk->next;
1999
2000 zend_mm_delete_chunk(heap, chunk);
2001 chunk = next_chunk;
2002 } else {
2003 chunk = chunk->next;
2004 }
2005 } while (chunk != heap->main_chunk);
2006
2007 return collected * ZEND_MM_PAGE_SIZE;
2008 }
2009
2010 #if ZEND_DEBUG
2011 /******************/
2012 /* Leak detection */
2013 /******************/
2014
2015 static zend_long zend_mm_find_leaks_small(zend_mm_chunk *p, uint32_t i, uint32_t j, zend_leak_info *leak)
2016 {
2017 int empty = 1;
2018 zend_long count = 0;
2019 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2020 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * (j + 1) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2021
2022 while (j < bin_elements[bin_num]) {
2023 if (dbg->size != 0) {
2024 if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2025 count++;
2026 dbg->size = 0;
2027 dbg->filename = NULL;
2028 dbg->lineno = 0;
2029 } else {
2030 empty = 0;
2031 }
2032 }
2033 j++;
2034 dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2035 }
2036 if (empty) {
2037 zend_mm_bitset_reset_range(p->free_map, i, bin_pages[bin_num]);
2038 }
2039 return count;
2040 }
2041
2042 static zend_long zend_mm_find_leaks(zend_mm_heap *heap, zend_mm_chunk *p, uint32_t i, zend_leak_info *leak)
2043 {
2044 zend_long count = 0;
2045
2046 do {
2047 while (i < p->free_tail) {
2048 if (zend_mm_bitset_is_set(p->free_map, i)) {
2049 if (p->map[i] & ZEND_MM_IS_SRUN) {
2050 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2051 count += zend_mm_find_leaks_small(p, i, 0, leak);
2052 i += bin_pages[bin_num];
2053 } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2054 int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2055 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2056
2057 if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) {
2058 count++;
2059 }
2060 zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2061 i += pages_count;
2062 }
2063 } else {
2064 i++;
2065 }
2066 }
2067 p = p->next;
2068 i = ZEND_MM_FIRST_PAGE;
2069 } while (p != heap->main_chunk);
2070 return count;
2071 }
2072
2073 static zend_long zend_mm_find_leaks_huge(zend_mm_heap *heap, zend_mm_huge_list *list)
2074 {
2075 zend_long count = 0;
2076 zend_mm_huge_list *prev = list;
2077 zend_mm_huge_list *p = list->next;
2078
2079 while (p) {
2080 if (p->dbg.filename == list->dbg.filename && p->dbg.lineno == list->dbg.lineno) {
2081 prev->next = p->next;
2082 zend_mm_chunk_free(heap, p->ptr, p->size);
2083 zend_mm_free_heap(heap, p, NULL, 0, NULL, 0);
2084 count++;
2085 } else {
2086 prev = p;
2087 }
2088 p = prev->next;
2089 }
2090
2091 return count;
2092 }
2093
2094 static void zend_mm_check_leaks(zend_mm_heap *heap)
2095 {
2096 zend_mm_huge_list *list;
2097 zend_mm_chunk *p;
2098 zend_leak_info leak;
2099 zend_long repeated = 0;
2100 uint32_t total = 0;
2101 uint32_t i, j;
2102
2103 /* find leaked huge blocks and free them */
2104 list = heap->huge_list;
2105 while (list) {
2106 zend_mm_huge_list *q = list;
2107
2108 leak.addr = list->ptr;
2109 leak.size = list->dbg.size;
2110 leak.filename = list->dbg.filename;
2111 leak.orig_filename = list->dbg.orig_filename;
2112 leak.lineno = list->dbg.lineno;
2113 leak.orig_lineno = list->dbg.orig_lineno;
2114
2115 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2116 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2117 repeated = zend_mm_find_leaks_huge(heap, list);
2118 total += 1 + repeated;
2119 if (repeated) {
2120 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2121 }
2122
2123 heap->huge_list = list = list->next;
2124 zend_mm_chunk_free(heap, q->ptr, q->size);
2125 zend_mm_free_heap(heap, q, NULL, 0, NULL, 0);
2126 }
2127
2128 /* for each chunk */
2129 p = heap->main_chunk;
2130 do {
2131 i = ZEND_MM_FIRST_PAGE;
2132 while (i < p->free_tail) {
2133 if (zend_mm_bitset_is_set(p->free_map, i)) {
2134 if (p->map[i] & ZEND_MM_IS_SRUN) {
2135 int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]);
2136 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2137
2138 j = 0;
2139 while (j < bin_elements[bin_num]) {
2140 if (dbg->size != 0) {
2141 leak.addr = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * j);
2142 leak.size = dbg->size;
2143 leak.filename = dbg->filename;
2144 leak.orig_filename = dbg->orig_filename;
2145 leak.lineno = dbg->lineno;
2146 leak.orig_lineno = dbg->orig_lineno;
2147
2148 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2149 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2150
2151 dbg->size = 0;
2152 dbg->filename = NULL;
2153 dbg->lineno = 0;
2154
2155 repeated = zend_mm_find_leaks_small(p, i, j + 1, &leak) +
2156 zend_mm_find_leaks(heap, p, i + bin_pages[bin_num], &leak);
2157 total += 1 + repeated;
2158 if (repeated) {
2159 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2160 }
2161 }
2162 dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]);
2163 j++;
2164 }
2165 i += bin_pages[bin_num];
2166 } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ {
2167 int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]);
2168 zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)));
2169
2170 leak.addr = (void*)((char*)p + ZEND_MM_PAGE_SIZE * i);
2171 leak.size = dbg->size;
2172 leak.filename = dbg->filename;
2173 leak.orig_filename = dbg->orig_filename;
2174 leak.lineno = dbg->lineno;
2175 leak.orig_lineno = dbg->orig_lineno;
2176
2177 zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL);
2178 zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak);
2179
2180 zend_mm_bitset_reset_range(p->free_map, i, pages_count);
2181
2182 repeated = zend_mm_find_leaks(heap, p, i + pages_count, &leak);
2183 total += 1 + repeated;
2184 if (repeated) {
2185 zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(zend_uintptr_t)repeated);
2186 }
2187 i += pages_count;
2188 }
2189 } else {
2190 i++;
2191 }
2192 }
2193 p = p->next;
2194 } while (p != heap->main_chunk);
2195 if (total) {
2196 zend_message_dispatcher(ZMSG_MEMORY_LEAKS_GRAND_TOTAL, &total);
2197 }
2198 }
2199 #endif
2200
2201 #if ZEND_MM_CUSTOM
2202 static void *tracked_malloc(size_t size);
2203 static void tracked_free_all();
2204 #endif
2205
2206 void zend_mm_shutdown(zend_mm_heap *heap, int full, int silent)
2207 {
2208 zend_mm_chunk *p;
2209 zend_mm_huge_list *list;
2210
2211 #if ZEND_MM_CUSTOM
2212 if (heap->use_custom_heap) {
2213 if (heap->custom_heap.std._malloc == tracked_malloc) {
2214 if (silent) {
2215 tracked_free_all();
2216 }
2217 zend_hash_clean(heap->tracked_allocs);
2218 if (full) {
2219 zend_hash_destroy(heap->tracked_allocs);
2220 free(heap->tracked_allocs);
2221 /* Make sure the heap free below does not use tracked_free(). */
2222 heap->custom_heap.std._free = free;
2223 }
2224 }
2225
2226 if (full) {
2227 if (ZEND_DEBUG && heap->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2228 heap->custom_heap.debug._free(heap ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC);
2229 } else {
2230 heap->custom_heap.std._free(heap);
2231 }
2232 }
2233 return;
2234 }
2235 #endif
2236
2237 #if ZEND_DEBUG
2238 if (!silent) {
2239 zend_mm_check_leaks(heap);
2240 }
2241 #endif
2242
2243 /* free huge blocks */
2244 list = heap->huge_list;
2245 heap->huge_list = NULL;
2246 while (list) {
2247 zend_mm_huge_list *q = list;
2248 list = list->next;
2249 zend_mm_chunk_free(heap, q->ptr, q->size);
2250 }
2251
2252 /* move all chunks except of the first one into the cache */
2253 p = heap->main_chunk->next;
2254 while (p != heap->main_chunk) {
2255 zend_mm_chunk *q = p->next;
2256 p->next = heap->cached_chunks;
2257 heap->cached_chunks = p;
2258 p = q;
2259 heap->chunks_count--;
2260 heap->cached_chunks_count++;
2261 }
2262
2263 if (full) {
2264 /* free all cached chunks */
2265 while (heap->cached_chunks) {
2266 p = heap->cached_chunks;
2267 heap->cached_chunks = p->next;
2268 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2269 }
2270 /* free the first chunk */
2271 zend_mm_chunk_free(heap, heap->main_chunk, ZEND_MM_CHUNK_SIZE);
2272 } else {
2273 /* free some cached chunks to keep average count */
2274 heap->avg_chunks_count = (heap->avg_chunks_count + (double)heap->peak_chunks_count) / 2.0;
2275 while ((double)heap->cached_chunks_count + 0.9 > heap->avg_chunks_count &&
2276 heap->cached_chunks) {
2277 p = heap->cached_chunks;
2278 heap->cached_chunks = p->next;
2279 zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE);
2280 heap->cached_chunks_count--;
2281 }
2282 /* clear cached chunks */
2283 p = heap->cached_chunks;
2284 while (p != NULL) {
2285 zend_mm_chunk *q = p->next;
2286 memset(p, 0, sizeof(zend_mm_chunk));
2287 p->next = q;
2288 p = q;
2289 }
2290
2291 /* reinitialize the first chunk and heap */
2292 p = heap->main_chunk;
2293 p->heap = &p->heap_slot;
2294 p->next = p;
2295 p->prev = p;
2296 p->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
2297 p->free_tail = ZEND_MM_FIRST_PAGE;
2298 p->num = 0;
2299
2300 #if ZEND_MM_STAT
2301 heap->size = heap->peak = 0;
2302 #endif
2303 memset(heap->free_slot, 0, sizeof(heap->free_slot));
2304 #if ZEND_MM_STAT || ZEND_MM_LIMIT
2305 heap->real_size = ZEND_MM_CHUNK_SIZE;
2306 #endif
2307 #if ZEND_MM_STAT
2308 heap->real_peak = ZEND_MM_CHUNK_SIZE;
2309 #endif
2310 heap->chunks_count = 1;
2311 heap->peak_chunks_count = 1;
2312 heap->last_chunks_delete_boundary = 0;
2313 heap->last_chunks_delete_count = 0;
2314
2315 memset(p->free_map, 0, sizeof(p->free_map) + sizeof(p->map));
2316 p->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1;
2317 p->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
2318 }
2319 }
2320
2321 /**************/
2322 /* PUBLIC API */
2323 /**************/
2324
2325 ZEND_API void* ZEND_FASTCALL _zend_mm_alloc(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2326 {
2327 return zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2328 }
2329
2330 ZEND_API void ZEND_FASTCALL _zend_mm_free(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2331 {
2332 zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2333 }
2334
2335 void* ZEND_FASTCALL _zend_mm_realloc(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2336 {
2337 return zend_mm_realloc_heap(heap, ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2338 }
2339
2340 void* ZEND_FASTCALL _zend_mm_realloc2(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2341 {
2342 return zend_mm_realloc_heap(heap, ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2343 }
2344
2345 ZEND_API size_t ZEND_FASTCALL _zend_mm_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2346 {
2347 return zend_mm_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2348 }
2349
2350 /**********************/
2351 /* Allocation Manager */
2352 /**********************/
2353
2354 typedef struct _zend_alloc_globals {
2355 zend_mm_heap *mm_heap;
2356 } zend_alloc_globals;
2357
2358 #ifdef ZTS
2359 static int alloc_globals_id;
2360 static size_t alloc_globals_offset;
2361 # define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v)
2362 #else
2363 # define AG(v) (alloc_globals.v)
2364 static zend_alloc_globals alloc_globals;
2365 #endif
2366
2367 ZEND_API int is_zend_mm(void)
2368 {
2369 #if ZEND_MM_CUSTOM
2370 return !AG(mm_heap)->use_custom_heap;
2371 #else
2372 return 1;
2373 #endif
2374 }
2375
2376 ZEND_API int is_zend_ptr(const void *ptr)
2377 {
2378 #if ZEND_MM_CUSTOM
2379 if (AG(mm_heap)->use_custom_heap) {
2380 return 0;
2381 }
2382 #endif
2383
2384 if (AG(mm_heap)->main_chunk) {
2385 zend_mm_chunk *chunk = AG(mm_heap)->main_chunk;
2386
2387 do {
2388 if (ptr >= (void*)chunk
2389 && ptr < (void*)((char*)chunk + ZEND_MM_CHUNK_SIZE)) {
2390 return 1;
2391 }
2392 chunk = chunk->next;
2393 } while (chunk != AG(mm_heap)->main_chunk);
2394 }
2395
2396 if (AG(mm_heap)->huge_list) {
2397 zend_mm_huge_list *block = AG(mm_heap)->huge_list;
2398
2399 do {
2400 if (ptr >= (void*)block
2401 && ptr < (void*)((char*)block + block->size)) {
2402 return 1;
2403 }
2404 block = block->next;
2405 } while (block != AG(mm_heap)->huge_list);
2406 }
2407 return 0;
2408 }
2409
2410 #if ZEND_MM_CUSTOM
2411
2412 static ZEND_COLD void* ZEND_FASTCALL _malloc_custom(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2413 {
2414 if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2415 return AG(mm_heap)->custom_heap.debug._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2416 } else {
2417 return AG(mm_heap)->custom_heap.std._malloc(size);
2418 }
2419 }
2420
2421 static ZEND_COLD void ZEND_FASTCALL _efree_custom(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2422 {
2423 if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2424 AG(mm_heap)->custom_heap.debug._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2425 } else {
2426 AG(mm_heap)->custom_heap.std._free(ptr);
2427 }
2428 }
2429
2430 static ZEND_COLD void* ZEND_FASTCALL _realloc_custom(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2431 {
2432 if (ZEND_DEBUG && AG(mm_heap)->use_custom_heap == ZEND_MM_CUSTOM_HEAP_DEBUG) {
2433 return AG(mm_heap)->custom_heap.debug._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2434 } else {
2435 return AG(mm_heap)->custom_heap.std._realloc(ptr, size);
2436 }
2437 }
2438 #endif
2439
2440 #if !ZEND_DEBUG && defined(HAVE_BUILTIN_CONSTANT_P)
2441 #undef _emalloc
2442
2443 #if ZEND_MM_CUSTOM
2444 # define ZEND_MM_CUSTOM_ALLOCATOR(size) do { \
2445 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2446 return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2447 } \
2448 } while (0)
2449 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr) do { \
2450 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \
2451 _efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2452 return; \
2453 } \
2454 } while (0)
2455 #else
2456 # define ZEND_MM_CUSTOM_ALLOCATOR(size)
2457 # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr)
2458 #endif
2459
2460 # define _ZEND_BIN_ALLOCATOR(_num, _size, _elements, _pages, x, y) \
2461 ZEND_API void* ZEND_FASTCALL _emalloc_ ## _size(void) { \
2462 ZEND_MM_CUSTOM_ALLOCATOR(_size); \
2463 return zend_mm_alloc_small(AG(mm_heap), _num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \
2464 }
2465
2466 ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR, x, y)
2467
2468 ZEND_API void* ZEND_FASTCALL _emalloc_large(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2469 {
2470 ZEND_MM_CUSTOM_ALLOCATOR(size);
2471 return zend_mm_alloc_large_ex(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2472 }
2473
2474 ZEND_API void* ZEND_FASTCALL _emalloc_huge(size_t size)
2475 {
2476 ZEND_MM_CUSTOM_ALLOCATOR(size);
2477 return zend_mm_alloc_huge(AG(mm_heap), size);
2478 }
2479
2480 #if ZEND_DEBUG
2481 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2482 ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2483 ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2484 { \
2485 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); \
2486 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2487 int page_num = page_offset / ZEND_MM_PAGE_SIZE; \
2488 ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2489 ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_SRUN); \
2490 ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(chunk->map[page_num]) == _num); \
2491 zend_mm_free_small(AG(mm_heap), ptr, _num); \
2492 } \
2493 }
2494 #else
2495 # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, x, y) \
2496 ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \
2497 ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \
2498 { \
2499 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \
2500 ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \
2501 zend_mm_free_small(AG(mm_heap), ptr, _num); \
2502 } \
2503 }
2504 #endif
2505
2506 ZEND_MM_BINS_INFO(_ZEND_BIN_FREE, x, y)
2507
2508 ZEND_API void ZEND_FASTCALL _efree_large(void *ptr, size_t size)
2509 {
2510 ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2511 {
2512 size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE);
2513 zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE);
2514 int page_num = page_offset / ZEND_MM_PAGE_SIZE;
2515 uint32_t pages_count = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE) / ZEND_MM_PAGE_SIZE;
2516
2517 ZEND_MM_CHECK(chunk->heap == AG(mm_heap) && ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted");
2518 ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_LRUN);
2519 ZEND_ASSERT(ZEND_MM_LRUN_PAGES(chunk->map[page_num]) == pages_count);
2520 zend_mm_free_large(AG(mm_heap), chunk, page_num, pages_count);
2521 }
2522 }
2523
2524 ZEND_API void ZEND_FASTCALL _efree_huge(void *ptr, size_t size)
2525 {
2526
2527 ZEND_MM_CUSTOM_DEALLOCATOR(ptr);
2528 zend_mm_free_huge(AG(mm_heap), ptr);
2529 }
2530 #endif
2531
2532 ZEND_API void* ZEND_FASTCALL _emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2533 {
2534 #if ZEND_MM_CUSTOM
2535 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2536 return _malloc_custom(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2537 }
2538 #endif
2539 return zend_mm_alloc_heap(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2540 }
2541
2542 ZEND_API void ZEND_FASTCALL _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2543 {
2544 #if ZEND_MM_CUSTOM
2545 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2546 _efree_custom(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2547 return;
2548 }
2549 #endif
2550 zend_mm_free_heap(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2551 }
2552
2553 ZEND_API void* ZEND_FASTCALL _erealloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2554 {
2555 #if ZEND_MM_CUSTOM
2556 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2557 return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2558 }
2559 #endif
2560 return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2561 }
2562
2563 ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2564 {
2565 #if ZEND_MM_CUSTOM
2566 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2567 return _realloc_custom(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2568 }
2569 #endif
2570 return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2571 }
2572
2573 ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2574 {
2575 #if ZEND_MM_CUSTOM
2576 if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) {
2577 return 0;
2578 }
2579 #endif
2580 return zend_mm_size(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2581 }
2582
2583 ZEND_API void* ZEND_FASTCALL _safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2584 {
2585 return _emalloc(zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2586 }
2587
2588 ZEND_API void* ZEND_FASTCALL _safe_malloc(size_t nmemb, size_t size, size_t offset)
2589 {
2590 return pemalloc(zend_safe_address_guarded(nmemb, size, offset), 1);
2591 }
2592
2593 ZEND_API void* ZEND_FASTCALL _safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2594 {
2595 return _erealloc(ptr, zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2596 }
2597
2598 ZEND_API void* ZEND_FASTCALL _safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset)
2599 {
2600 return perealloc(ptr, zend_safe_address_guarded(nmemb, size, offset), 1);
2601 }
2602
2603 ZEND_API void* ZEND_FASTCALL _ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2604 {
2605 void *p;
2606
2607 size = zend_safe_address_guarded(nmemb, size, 0);
2608 p = _emalloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2609 memset(p, 0, size);
2610 return p;
2611 }
2612
2613 ZEND_API char* ZEND_FASTCALL _estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2614 {
2615 size_t length;
2616 char *p;
2617
2618 length = strlen(s);
2619 if (UNEXPECTED(length + 1 == 0)) {
2620 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2621 }
2622 p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2623 memcpy(p, s, length+1);
2624 return p;
2625 }
2626
2627 ZEND_API char* ZEND_FASTCALL _estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)
2628 {
2629 char *p;
2630
2631 if (UNEXPECTED(length + 1 == 0)) {
2632 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2633 }
2634 p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC);
2635 memcpy(p, s, length);
2636 p[length] = 0;
2637 return p;
2638 }
2639
2640
2641 ZEND_API char* ZEND_FASTCALL zend_strndup(const char *s, size_t length)
2642 {
2643 char *p;
2644
2645 if (UNEXPECTED(length + 1 == 0)) {
2646 zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length);
2647 }
2648 p = (char *) malloc(length + 1);
2649 if (UNEXPECTED(p == NULL)) {
2650 return p;
2651 }
2652 if (EXPECTED(length)) {
2653 memcpy(p, s, length);
2654 }
2655 p[length] = 0;
2656 return p;
2657 }
2658
2659
2660 ZEND_API int zend_set_memory_limit(size_t memory_limit)
2661 {
2662 #if ZEND_MM_LIMIT
2663 if (memory_limit < ZEND_MM_CHUNK_SIZE) {
2664 memory_limit = ZEND_MM_CHUNK_SIZE;
2665 }
2666 if (UNEXPECTED(memory_limit < AG(mm_heap)->real_size)) {
2667 return FAILURE;
2668 }
2669 AG(mm_heap)->limit = memory_limit;
2670 #endif
2671 return SUCCESS;
2672 }
2673
2674 ZEND_API size_t zend_memory_usage(int real_usage)
2675 {
2676 #if ZEND_MM_STAT
2677 if (real_usage) {
2678 return AG(mm_heap)->real_size;
2679 } else {
2680 size_t usage = AG(mm_heap)->size;
2681 return usage;
2682 }
2683 #endif
2684 return 0;
2685 }
2686
2687 ZEND_API size_t zend_memory_peak_usage(int real_usage)
2688 {
2689 #if ZEND_MM_STAT
2690 if (real_usage) {
2691 return AG(mm_heap)->real_peak;
2692 } else {
2693 return AG(mm_heap)->peak;
2694 }
2695 #endif
2696 return 0;
2697 }
2698
2699 ZEND_API void shutdown_memory_manager(int silent, int full_shutdown)
2700 {
2701 zend_mm_shutdown(AG(mm_heap), full_shutdown, silent);
2702 }
2703
2704 #if ZEND_MM_CUSTOM
2705 static void *tracked_malloc(size_t size)
2706 {
2707 void *ptr = __zend_malloc(size);
2708 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2709 ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr);
2710 zend_hash_index_add_empty_element(AG(mm_heap)->tracked_allocs, h);
2711 return ptr;
2712 }
2713
2714 static void tracked_free(void *ptr) {
2715 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2716 zend_hash_index_del(AG(mm_heap)->tracked_allocs, h);
2717 free(ptr);
2718 }
2719
2720 static void *tracked_realloc(void *ptr, size_t new_size) {
2721 zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2722 zend_hash_index_del(AG(mm_heap)->tracked_allocs, h);
2723 ptr = __zend_realloc(ptr, new_size);
2724 h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2;
2725 ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr);
2726 zend_hash_index_add_empty_element(AG(mm_heap)->tracked_allocs, h);
2727 return ptr;
2728 }
2729
2730 static void tracked_free_all() {
2731 HashTable *tracked_allocs = AG(mm_heap)->tracked_allocs;
2732 zend_ulong h;
2733 ZEND_HASH_FOREACH_NUM_KEY(tracked_allocs, h) {
2734 void *ptr = (void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2);
2735 free(ptr);
2736 } ZEND_HASH_FOREACH_END();
2737 }
2738 #endif
2739
2740 static void alloc_globals_ctor(zend_alloc_globals *alloc_globals)
2741 {
2742 char *tmp;
2743
2744 #if ZEND_MM_CUSTOM
2745 tmp = getenv("USE_ZEND_ALLOC");
2746 if (tmp && !zend_atoi(tmp, 0)) {
2747 zend_bool tracked = (tmp = getenv("USE_TRACKED_ALLOC")) && zend_atoi(tmp, 0);
2748 zend_mm_heap *mm_heap = alloc_globals->mm_heap = malloc(sizeof(zend_mm_heap));
2749 memset(mm_heap, 0, sizeof(zend_mm_heap));
2750 mm_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2751 if (!tracked) {
2752 /* Use system allocator. */
2753 mm_heap->custom_heap.std._malloc = __zend_malloc;
2754 mm_heap->custom_heap.std._free = free;
2755 mm_heap->custom_heap.std._realloc = __zend_realloc;
2756 } else {
2757 /* Use system allocator and track allocations for auto-free. */
2758 mm_heap->custom_heap.std._malloc = tracked_malloc;
2759 mm_heap->custom_heap.std._free = tracked_free;
2760 mm_heap->custom_heap.std._realloc = tracked_realloc;
2761 mm_heap->tracked_allocs = malloc(sizeof(HashTable));
2762 zend_hash_init(mm_heap->tracked_allocs, 1024, NULL, NULL, 1);
2763 }
2764 return;
2765 }
2766 #endif
2767
2768 tmp = getenv("USE_ZEND_ALLOC_HUGE_PAGES");
2769 if (tmp && zend_atoi(tmp, 0)) {
2770 zend_mm_use_huge_pages = 1;
2771 }
2772 alloc_globals->mm_heap = zend_mm_init();
2773 }
2774
2775 #ifdef ZTS
2776 static void alloc_globals_dtor(zend_alloc_globals *alloc_globals)
2777 {
2778 zend_mm_shutdown(alloc_globals->mm_heap, 1, 1);
2779 }
2780 #endif
2781
2782 ZEND_API void start_memory_manager(void)
2783 {
2784 #ifdef ZTS
2785 ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor);
2786 #else
2787 alloc_globals_ctor(&alloc_globals);
2788 #endif
2789 #ifndef _WIN32
2790 # if defined(_SC_PAGESIZE)
2791 REAL_PAGE_SIZE = sysconf(_SC_PAGESIZE);
2792 # elif defined(_SC_PAGE_SIZE)
2793 REAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE);
2794 # endif
2795 #endif
2796 }
2797
2798 ZEND_API zend_mm_heap *zend_mm_set_heap(zend_mm_heap *new_heap)
2799 {
2800 zend_mm_heap *old_heap;
2801
2802 old_heap = AG(mm_heap);
2803 AG(mm_heap) = (zend_mm_heap*)new_heap;
2804 return (zend_mm_heap*)old_heap;
2805 }
2806
2807 ZEND_API zend_mm_heap *zend_mm_get_heap(void)
2808 {
2809 return AG(mm_heap);
2810 }
2811
2812 ZEND_API int zend_mm_is_custom_heap(zend_mm_heap *new_heap)
2813 {
2814 #if ZEND_MM_CUSTOM
2815 return AG(mm_heap)->use_custom_heap;
2816 #else
2817 return 0;
2818 #endif
2819 }
2820
2821 ZEND_API void zend_mm_set_custom_handlers(zend_mm_heap *heap,
2822 void* (*_malloc)(size_t),
2823 void (*_free)(void*),
2824 void* (*_realloc)(void*, size_t))
2825 {
2826 #if ZEND_MM_CUSTOM
2827 zend_mm_heap *_heap = (zend_mm_heap*)heap;
2828
2829 if (!_malloc && !_free && !_realloc) {
2830 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE;
2831 } else {
2832 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD;
2833 _heap->custom_heap.std._malloc = _malloc;
2834 _heap->custom_heap.std._free = _free;
2835 _heap->custom_heap.std._realloc = _realloc;
2836 }
2837 #endif
2838 }
2839
2840 ZEND_API void zend_mm_get_custom_handlers(zend_mm_heap *heap,
2841 void* (**_malloc)(size_t),
2842 void (**_free)(void*),
2843 void* (**_realloc)(void*, size_t))
2844 {
2845 #if ZEND_MM_CUSTOM
2846 zend_mm_heap *_heap = (zend_mm_heap*)heap;
2847
2848 if (heap->use_custom_heap) {
2849 *_malloc = _heap->custom_heap.std._malloc;
2850 *_free = _heap->custom_heap.std._free;
2851 *_realloc = _heap->custom_heap.std._realloc;
2852 } else {
2853 *_malloc = NULL;
2854 *_free = NULL;
2855 *_realloc = NULL;
2856 }
2857 #else
2858 *_malloc = NULL;
2859 *_free = NULL;
2860 *_realloc = NULL;
2861 #endif
2862 }
2863
2864 #if ZEND_DEBUG
2865 ZEND_API void zend_mm_set_custom_debug_handlers(zend_mm_heap *heap,
2866 void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2867 void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC),
2868 void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC))
2869 {
2870 #if ZEND_MM_CUSTOM
2871 zend_mm_heap *_heap = (zend_mm_heap*)heap;
2872
2873 _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_DEBUG;
2874 _heap->custom_heap.debug._malloc = _malloc;
2875 _heap->custom_heap.debug._free = _free;
2876 _heap->custom_heap.debug._realloc = _realloc;
2877 #endif
2878 }
2879 #endif
2880
2881 ZEND_API zend_mm_storage *zend_mm_get_storage(zend_mm_heap *heap)
2882 {
2883 #if ZEND_MM_STORAGE
2884 return heap->storage;
2885 #else
2886 return NULL
2887 #endif
2888 }
2889
2890 ZEND_API zend_mm_heap *zend_mm_startup(void)
2891 {
2892 return zend_mm_init();
2893 }
2894
2895 ZEND_API zend_mm_heap *zend_mm_startup_ex(const zend_mm_handlers *handlers, void *data, size_t data_size)
2896 {
2897 #if ZEND_MM_STORAGE
2898 zend_mm_storage tmp_storage, *storage;
2899 zend_mm_chunk *chunk;
2900 zend_mm_heap *heap;
2901
2902 memcpy((zend_mm_handlers*)&tmp_storage.handlers, handlers, sizeof(zend_mm_handlers));
2903 tmp_storage.data = data;
2904 chunk = (zend_mm_chunk*)handlers->chunk_alloc(&tmp_storage, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE);
2905 if (UNEXPECTED(chunk == NULL)) {
2906 #if ZEND_MM_ERROR
2907 #ifdef _WIN32
2908 stderr_last_error("Can't initialize heap");
2909 #else
2910 fprintf(stderr, "\nCan't initialize heap: [%d] %s\n", errno, strerror(errno));
2911 #endif
2912 #endif
2913 return NULL;
2914 }
2915 heap = &chunk->heap_slot;
2916 chunk->heap = heap;
2917 chunk->next = chunk;
2918 chunk->prev = chunk;
2919 chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE;
2920 chunk->free_tail = ZEND_MM_FIRST_PAGE;
2921 chunk->num = 0;
2922 chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1;
2923 chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE);
2924 heap->main_chunk = chunk;
2925 heap->cached_chunks = NULL;
2926 heap->chunks_count = 1;
2927 heap->peak_chunks_count = 1;
2928 heap->cached_chunks_count = 0;
2929 heap->avg_chunks_count = 1.0;
2930 heap->last_chunks_delete_boundary = 0;
2931 heap->last_chunks_delete_count = 0;
2932 #if ZEND_MM_STAT || ZEND_MM_LIMIT
2933 heap->real_size = ZEND_MM_CHUNK_SIZE;
2934 #endif
2935 #if ZEND_MM_STAT
2936 heap->real_peak = ZEND_MM_CHUNK_SIZE;
2937 heap->size = 0;
2938 heap->peak = 0;
2939 #endif
2940 #if ZEND_MM_LIMIT
2941 heap->limit = (Z_L(-1) >> Z_L(1));
2942 heap->overflow = 0;
2943 #endif
2944 #if ZEND_MM_CUSTOM
2945 heap->use_custom_heap = 0;
2946 #endif
2947 heap->storage = &tmp_storage;
2948 heap->huge_list = NULL;
2949 memset(heap->free_slot, 0, sizeof(heap->free_slot));
2950 storage = _zend_mm_alloc(heap, sizeof(zend_mm_storage) + data_size ZEND_FILE_LINE_CC ZEND_FILE_LINE_CC);
2951 if (!storage) {
2952 handlers->chunk_free(&tmp_storage, chunk, ZEND_MM_CHUNK_SIZE);
2953 #if ZEND_MM_ERROR
2954 #ifdef _WIN32
2955 stderr_last_error("Can't initialize heap");
2956 #else
2957 fprintf(stderr, "\nCan't initialize heap: [%d] %s\n", errno, strerror(errno));
2958 #endif
2959 #endif
2960 return NULL;
2961 }
2962 memcpy(storage, &tmp_storage, sizeof(zend_mm_storage));
2963 if (data) {
2964 storage->data = (void*)(((char*)storage + sizeof(zend_mm_storage)));
2965 memcpy(storage->data, data, data_size);
2966 }
2967 heap->storage = storage;
2968 return heap;
2969 #else
2970 return NULL;
2971 #endif
2972 }
2973
2974 static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void)
2975 {
2976 fprintf(stderr, "Out of memory\n");
2977 exit(1);
2978 }
2979
2980 ZEND_API void * __zend_malloc(size_t len)
2981 {
2982 void *tmp = malloc(len);
2983 if (EXPECTED(tmp || !len)) {
2984 return tmp;
2985 }
2986 zend_out_of_memory();
2987 }
2988
2989 ZEND_API void * __zend_calloc(size_t nmemb, size_t len)
2990 {
2991 void *tmp;
2992
2993 len = zend_safe_address_guarded(nmemb, len, 0);
2994 tmp = __zend_malloc(len);
2995 memset(tmp, 0, len);
2996 return tmp;
2997 }
2998
2999 ZEND_API void * __zend_realloc(void *p, size_t len)
3000 {
3001 p = realloc(p, len);
3002 if (EXPECTED(p || !len)) {
3003 return p;
3004 }
3005 zend_out_of_memory();
3006 }
3007
3008 #ifdef ZTS
3009 size_t zend_mm_globals_size(void)
3010 {
3011 return sizeof(zend_alloc_globals);
3012 }
3013 #endif
3014
