1 /*
2 +----------------------------------------------------------------------+
3 | Zend OPcache |
4 +----------------------------------------------------------------------+
5 | Copyright (c) The PHP Group |
6 +----------------------------------------------------------------------+
7 | This source file is subject to version 3.01 of the PHP license, |
8 | that is bundled with this package in the file LICENSE, and is |
9 | available through the world-wide-web at the following url: |
10 | https://www.php.net/license/3_01.txt |
11 | If you did not receive a copy of the PHP license and are unable to |
12 | obtain it through the world-wide-web, please send a note to |
13 | license@php.net so we can mail you a copy immediately. |
14 +----------------------------------------------------------------------+
15 | Authors: Andi Gutmans <andi@php.net> |
16 | Zeev Suraski <zeev@php.net> |
17 | Stanislav Malyshev <stas@zend.com> |
18 | Dmitry Stogov <dmitry@php.net> |
19 +----------------------------------------------------------------------+
20 */
21
22 #include "zend_shared_alloc.h"
23
24 #ifdef USE_MMAP
25
26 #include <sys/types.h>
27 #include <sys/stat.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <sys/mman.h>
31
32 #ifdef __APPLE__
33 #include <mach/vm_statistics.h>
34 #endif
35
36 #include "zend_execute.h"
37 #ifdef HAVE_PROCCTL
38 #include <sys/procctl.h>
39 #endif
40
41 #if defined(MAP_ANON) && !defined(MAP_ANONYMOUS)
42 # define MAP_ANONYMOUS MAP_ANON
43 #endif
44 #if defined(MAP_ALIGNED_SUPER)
45 # include <sys/types.h>
46 # include <sys/sysctl.h>
47 # include <sys/user.h>
48 # define MAP_HUGETLB MAP_ALIGNED_SUPER
49 #endif
50
51 #if (defined(__linux__) || defined(__FreeBSD__)) && (defined(__x86_64__) || defined (__aarch64__)) && !defined(__SANITIZE_ADDRESS__)
find_prefered_mmap_base(size_t requested_size)52 static void *find_prefered_mmap_base(size_t requested_size)
53 {
54 size_t huge_page_size = 2 * 1024 * 1024;
55 uintptr_t last_free_addr = huge_page_size;
56 uintptr_t last_candidate = (uintptr_t)MAP_FAILED;
57 uintptr_t start, end, text_start = 0;
58 #if defined(__linux__)
59 FILE *f;
60 char buffer[MAXPATHLEN];
61
62 f = fopen("/proc/self/maps", "r");
63 if (!f) {
64 return MAP_FAILED;
65 }
66
67 while (fgets(buffer, MAXPATHLEN, f) && sscanf(buffer, "%lx-%lx", &start, &end) == 2) {
68 /* Don't place the segment directly before or after the heap segment. Due to an selinux bug,
69 * a segment directly preceding or following the heap is interpreted as heap memory, which
70 * will result in an execheap violation for the JIT.
71 * See https://bugzilla.kernel.org/show_bug.cgi?id=218258. */
72 bool heap_segment = strstr(buffer, "[heap]") != NULL;
73 if (heap_segment) {
74 uintptr_t start_base = start & ~(huge_page_size - 1);
75 if (last_free_addr + requested_size >= start_base) {
76 last_free_addr = ZEND_MM_ALIGNED_SIZE_EX(end + huge_page_size, huge_page_size);
77 continue;
78 }
79 }
80 if ((uintptr_t)execute_ex >= start) {
81 /* the current segment lays before PHP .text segment or PHP .text segment itself */
82 /*Search for candidates at the end of the free segment near the .text segment
83 to prevent candidates from being missed due to large hole*/
84 if (last_free_addr + requested_size <= start) {
85 last_candidate = ZEND_MM_ALIGNED_SIZE_EX(start - requested_size, huge_page_size);
86 if (last_candidate + requested_size > start) {
87 last_candidate -= huge_page_size;
88 }
89 }
90 if ((uintptr_t)execute_ex < end) {
91 /* the current segment is PHP .text segment itself */
92 if (last_candidate != (uintptr_t)MAP_FAILED) {
93 if (end - last_candidate < UINT32_MAX) {
94 /* we have found a big enough hole before the text segment */
95 break;
96 }
97 last_candidate = (uintptr_t)MAP_FAILED;
98 }
99 text_start = start;
100 }
101 } else {
102 /* the current segment lays after PHP .text segment */
103 if (last_free_addr + requested_size - text_start > UINT32_MAX) {
104 /* the current segment and the following segments lay too far from PHP .text segment */
105 break;
106 }
107 if (last_free_addr + requested_size <= start) {
108 last_candidate = last_free_addr;
109 break;
110 }
111 }
112 last_free_addr = ZEND_MM_ALIGNED_SIZE_EX(end, huge_page_size);
113 if (heap_segment) {
114 last_free_addr += huge_page_size;
115 }
116 }
117 fclose(f);
118 #elif defined(__FreeBSD__)
119 size_t s = 0;
120 int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
121 if (sysctl(mib, 4, NULL, &s, NULL, 0) == 0) {
122 s = s * 4 / 3;
123 void *addr = mmap(NULL, s, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0);
124 if (addr != MAP_FAILED) {
125 if (sysctl(mib, 4, addr, &s, NULL, 0) == 0) {
126 start = (uintptr_t)addr;
127 end = start + s;
128 while (start < end) {
129 struct kinfo_vmentry *entry = (struct kinfo_vmentry *)start;
130 size_t sz = entry->kve_structsize;
131 if (sz == 0) {
132 break;
133 }
134 uintptr_t e_start = entry->kve_start;
135 uintptr_t e_end = entry->kve_end;
136 if ((uintptr_t)execute_ex >= e_start) {
137 /* the current segment lays before PHP .text segment or PHP .text segment itself */
138 if (last_free_addr + requested_size <= e_start) {
139 last_candidate = ZEND_MM_ALIGNED_SIZE_EX(e_start - requested_size, huge_page_size);
140 if (last_candidate + requested_size > e_start) {
141 last_candidate -= huge_page_size;
142 }
143 }
144 if ((uintptr_t)execute_ex < e_end) {
145 /* the current segment is PHP .text segment itself */
146 if (last_candidate != (uintptr_t)MAP_FAILED) {
147 if (e_end - last_candidate < UINT32_MAX) {
148 /* we have found a big enough hole before the text segment */
149 break;
150 }
151 last_candidate = (uintptr_t)MAP_FAILED;
152 }
153 text_start = e_start;
154 }
155 } else {
156 /* the current segment lays after PHP .text segment */
157 if (last_free_addr + requested_size - text_start > UINT32_MAX) {
158 /* the current segment and the following segments lay too far from PHP .text segment */
159 break;
160 }
161 if (last_free_addr + requested_size <= e_start) {
162 last_candidate = last_free_addr;
163 break;
164 }
165 }
166 last_free_addr = ZEND_MM_ALIGNED_SIZE_EX(e_end, huge_page_size);
167 start += sz;
168 }
169 }
170 munmap(addr, s);
171 }
172 }
173 #endif
174
175 return (void*)last_candidate;
176 }
177 #endif
178
create_segments(size_t requested_size,zend_shared_segment *** shared_segments_p,int * shared_segments_count,const char ** error_in)179 static int create_segments(size_t requested_size, zend_shared_segment ***shared_segments_p, int *shared_segments_count, const char **error_in)
180 {
181 zend_shared_segment *shared_segment;
182 int flags = PROT_READ | PROT_WRITE, fd = -1;
183 void *p;
184 #if defined(HAVE_PROCCTL) && defined(PROC_WXMAP_CTL)
185 int enable_wxmap = PROC_WX_MAPPINGS_PERMIT;
186 if (procctl(P_PID, getpid(), PROC_WXMAP_CTL, &enable_wxmap) == -1) {
187 return ALLOC_FAILURE;
188 }
189 #endif
190 #ifdef PROT_MPROTECT
191 flags |= PROT_MPROTECT(PROT_EXEC);
192 #endif
193 #ifdef VM_MAKE_TAG
194 /* allows tracking segments via tools such as vmmap */
195 fd = VM_MAKE_TAG(251U);
196 #endif
197 #ifdef PROT_MAX
198 flags |= PROT_MAX(PROT_READ | PROT_WRITE | PROT_EXEC);
199 #endif
200 #if (defined(__linux__) || defined(__FreeBSD__)) && (defined(__x86_64__) || defined (__aarch64__)) && !defined(__SANITIZE_ADDRESS__)
201 void *hint = find_prefered_mmap_base(requested_size);
202 if (hint != MAP_FAILED) {
203 # ifdef MAP_HUGETLB
204 size_t huge_page_size = 2 * 1024 * 1024;
205 if (requested_size >= huge_page_size && requested_size % huge_page_size == 0) {
206 p = mmap(hint, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB|MAP_FIXED, -1, 0);
207 if (p != MAP_FAILED) {
208 goto success;
209 }
210 }
211 #endif
212 p = mmap(hint, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
213 if (p != MAP_FAILED) {
214 goto success;
215 }
216 }
217 #endif
218 #ifdef MAP_HUGETLB
219 size_t huge_page_size = 2 * 1024 * 1024;
220
221 /* Try to allocate huge pages first to reduce dTLB misses.
222 * OSes has to be configured properly
223 * on Linux
224 * (e.g. https://wiki.debian.org/Hugepages#Enabling_HugeTlbPage)
225 * You may verify huge page usage with the following command:
226 * `grep "Huge" /proc/meminfo`
227 * on FreeBSD
228 * sysctl vm.pmap.pg_ps_enabled entry
229 * (boot time config only, but enabled by default on most arches).
230 */
231 if (requested_size >= huge_page_size && requested_size % huge_page_size == 0) {
232 # if defined(__x86_64__) && defined(MAP_32BIT)
233 /* to got HUGE PAGES in low 32-bit address we have to reserve address
234 space and then remap it using MAP_HUGETLB */
235
236 p = mmap(NULL, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, fd, 0);
237 if (p != MAP_FAILED) {
238 munmap(p, requested_size);
239 p = (void*)(ZEND_MM_ALIGNED_SIZE_EX((ptrdiff_t)p, huge_page_size));
240 p = mmap(p, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT|MAP_HUGETLB|MAP_FIXED, -1, 0);
241 if (p != MAP_FAILED) {
242 goto success;
243 } else {
244 p = mmap(NULL, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, fd, 0);
245 if (p != MAP_FAILED) {
246 goto success;
247 }
248 }
249 }
250 # endif
251 p = mmap(0, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB, fd, 0);
252 if (p != MAP_FAILED) {
253 goto success;
254 }
255 }
256 #elif defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT)
257 p = mmap(NULL, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, fd, 0);
258 if (p != MAP_FAILED) {
259 goto success;
260 }
261 #endif
262
263 p = mmap(0, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS, fd, 0);
264 if (p == MAP_FAILED) {
265 *error_in = "mmap";
266 return ALLOC_FAILURE;
267 }
268
269 success: ZEND_ATTRIBUTE_UNUSED;
270 *shared_segments_count = 1;
271 *shared_segments_p = (zend_shared_segment **) calloc(1, sizeof(zend_shared_segment) + sizeof(void *));
272 if (!*shared_segments_p) {
273 munmap(p, requested_size);
274 *error_in = "calloc";
275 return ALLOC_FAILURE;
276 }
277 shared_segment = (zend_shared_segment *)((char *)(*shared_segments_p) + sizeof(void *));
278 (*shared_segments_p)[0] = shared_segment;
279
280 shared_segment->p = p;
281 shared_segment->pos = 0;
282 shared_segment->size = requested_size;
283
284 return ALLOC_SUCCESS;
285 }
286
detach_segment(zend_shared_segment * shared_segment)287 static int detach_segment(zend_shared_segment *shared_segment)
288 {
289 munmap(shared_segment->p, shared_segment->size);
290 return 0;
291 }
292
segment_type_size(void)293 static size_t segment_type_size(void)
294 {
295 return sizeof(zend_shared_segment);
296 }
297
298 const zend_shared_memory_handlers zend_alloc_mmap_handlers = {
299 create_segments,
300 detach_segment,
301 segment_type_size
302 };
303
304 #endif /* USE_MMAP */
305
