xref: /PHP-8.2/ext/opcache/shared_alloc_mmap.c (revision acf758d2)
1 /*
2    +----------------------------------------------------------------------+
3    | Zend OPcache                                                         |
4    +----------------------------------------------------------------------+
5    | Copyright (c) The PHP Group                                          |
6    +----------------------------------------------------------------------+
7    | This source file is subject to version 3.01 of the PHP license,      |
8    | that is bundled with this package in the file LICENSE, and is        |
9    | available through the world-wide-web at the following url:           |
10    | https://www.php.net/license/3_01.txt                                 |
11    | If you did not receive a copy of the PHP license and are unable to   |
12    | obtain it through the world-wide-web, please send a note to          |
13    | license@php.net so we can mail you a copy immediately.               |
14    +----------------------------------------------------------------------+
15    | Authors: Andi Gutmans <andi@php.net>                                 |
16    |          Zeev Suraski <zeev@php.net>                                 |
17    |          Stanislav Malyshev <stas@zend.com>                          |
18    |          Dmitry Stogov <dmitry@php.net>                              |
19    +----------------------------------------------------------------------+
20 */
21 
22 #include "zend_shared_alloc.h"
23 #ifdef HAVE_JIT
24 # include "jit/zend_jit.h"
25 #endif
26 
27 #ifdef USE_MMAP
28 
29 #include <sys/types.h>
30 #include <sys/stat.h>
31 #include <stdio.h>
32 #include <stdlib.h>
33 #include <sys/mman.h>
34 
35 #ifdef __APPLE__
36 #include <mach/vm_statistics.h>
37 #endif
38 
39 #include "zend_execute.h"
40 
41 #if defined(MAP_ANON) && !defined(MAP_ANONYMOUS)
42 # define MAP_ANONYMOUS MAP_ANON
43 #endif
44 #if defined(MAP_ALIGNED_SUPER)
45 # include <sys/types.h>
46 # include <sys/sysctl.h>
47 # include <sys/user.h>
48 # define MAP_HUGETLB MAP_ALIGNED_SUPER
49 #endif
50 
51 #if defined(HAVE_JIT) && (defined(__linux__) || defined(__FreeBSD__)) && (defined(__x86_64__) || defined (__aarch64__)) && !defined(__SANITIZE_ADDRESS__)
find_prefered_mmap_base(size_t requested_size)52 static void *find_prefered_mmap_base(size_t requested_size)
53 {
54 	size_t huge_page_size = 2 * 1024 * 1024;
55 	uintptr_t last_free_addr = huge_page_size;
56 	uintptr_t last_candidate = (uintptr_t)MAP_FAILED;
57 	uintptr_t start, end, text_start = 0;
58 #if defined(__linux__)
59 	FILE *f;
60 	char buffer[MAXPATHLEN];
61 
62 	f = fopen("/proc/self/maps", "r");
63 	if (!f) {
64 		return MAP_FAILED;
65 	}
66 
67 	while (fgets(buffer, MAXPATHLEN, f) && sscanf(buffer, "%lx-%lx", &start, &end) == 2) {
68 		/* Don't place the segment directly before or after the heap segment. Due to an selinux bug,
69 		 * a segment directly preceding or following the heap is interpreted as heap memory, which
70 		 * will result in an execheap violation for the JIT.
71 		 * See https://bugzilla.kernel.org/show_bug.cgi?id=218258. */
72 		bool heap_segment = strstr(buffer, "[heap]") != NULL;
73 		if (heap_segment) {
74 			uintptr_t start_base = start & ~(huge_page_size - 1);
75 			if (last_free_addr + requested_size >= start_base) {
76 				last_free_addr = ZEND_MM_ALIGNED_SIZE_EX(end + huge_page_size, huge_page_size);
77 				continue;
78 			}
79 		}
80 		if ((uintptr_t)execute_ex >= start) {
81 			/* the current segment lays before PHP .text segment or PHP .text segment itself */
82 			/*Search for candidates at the end of the free segment near the .text segment
83 			  to prevent candidates from being missed due to large hole*/
84 			if (last_free_addr + requested_size <= start) {
85 				last_candidate = ZEND_MM_ALIGNED_SIZE_EX(start - requested_size, huge_page_size);
86 				if (last_candidate + requested_size > start) {
87 					last_candidate -= huge_page_size;
88 				}
89 			}
90 			if ((uintptr_t)execute_ex < end) {
91 				/* the current segment is PHP .text segment itself */
92 				if (last_candidate != (uintptr_t)MAP_FAILED) {
93 					if (end - last_candidate < UINT32_MAX) {
94 						/* we have found a big enough hole before the text segment */
95 						break;
96 					}
97 					last_candidate = (uintptr_t)MAP_FAILED;
98 				}
99 				text_start = start;
100 			}
101 		} else {
102 			/* the current segment lays after PHP .text segment */
103 			if (last_free_addr + requested_size - text_start > UINT32_MAX) {
104 				/* the current segment and the following segments lay too far from PHP .text segment */
105 				break;
106 			}
107 			if (last_free_addr + requested_size <= start) {
108 				last_candidate = last_free_addr;
109 				break;
110 			}
111 		}
112 		last_free_addr = ZEND_MM_ALIGNED_SIZE_EX(end, huge_page_size);
113 		if (heap_segment) {
114 			last_free_addr += huge_page_size;
115 		}
116 	}
117 	fclose(f);
118 #elif defined(__FreeBSD__)
119 	size_t s = 0;
120 	int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
121 	if (sysctl(mib, 4, NULL, &s, NULL, 0) == 0) {
122 		s = s * 4 / 3;
123 		void *addr = mmap(NULL, s, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, -1, 0);
124 		if (addr != MAP_FAILED) {
125 			if (sysctl(mib, 4, addr, &s, NULL, 0) == 0) {
126 				start = (uintptr_t)addr;
127 				end = start + s;
128 				while (start < end) {
129 					struct kinfo_vmentry *entry = (struct kinfo_vmentry *)start;
130 					size_t sz = entry->kve_structsize;
131 					if (sz == 0) {
132 						break;
133 					}
134 					uintptr_t e_start = entry->kve_start;
135 					uintptr_t e_end = entry->kve_end;
136 					if ((uintptr_t)execute_ex >= e_start) {
137 						/* the current segment lays before PHP .text segment or PHP .text segment itself */
138 						if (last_free_addr + requested_size <= e_start) {
139 							last_candidate = ZEND_MM_ALIGNED_SIZE_EX(e_start - requested_size, huge_page_size);
140 							if (last_candidate + requested_size > e_start) {
141 								last_candidate -= huge_page_size;
142 							}
143 						}
144 						if ((uintptr_t)execute_ex < e_end) {
145 							/* the current segment is PHP .text segment itself */
146 							if (last_candidate != (uintptr_t)MAP_FAILED) {
147 								if (e_end - last_candidate < UINT32_MAX) {
148 									/* we have found a big enough hole before the text segment */
149 									break;
150 								}
151 								last_candidate = (uintptr_t)MAP_FAILED;
152 							}
153 							text_start = e_start;
154 						}
155 					} else {
156 						/* the current segment lays after PHP .text segment */
157 						if (last_free_addr + requested_size - text_start > UINT32_MAX) {
158 							/* the current segment and the following segments lay too far from PHP .text segment */
159 							break;
160 						}
161 						if (last_free_addr + requested_size <= e_start) {
162 							last_candidate = last_free_addr;
163 							break;
164 						}
165 					}
166 					last_free_addr = ZEND_MM_ALIGNED_SIZE_EX(e_end, huge_page_size);
167 					start += sz;
168 				}
169 			}
170 			munmap(addr, s);
171 		}
172 	}
173 #endif
174 
175 	return (void*)last_candidate;
176 }
177 #endif
178 
create_segments(size_t requested_size,zend_shared_segment *** shared_segments_p,int * shared_segments_count,char ** error_in)179 static int create_segments(size_t requested_size, zend_shared_segment ***shared_segments_p, int *shared_segments_count, char **error_in)
180 {
181 	zend_shared_segment *shared_segment;
182 	int flags = PROT_READ | PROT_WRITE, fd = -1;
183 	void *p;
184 #ifdef PROT_MPROTECT
185 	flags |= PROT_MPROTECT(PROT_EXEC);
186 #endif
187 #ifdef VM_MAKE_TAG
188 	/* allows tracking segments via tools such as vmmap */
189 	fd = VM_MAKE_TAG(251U);
190 #endif
191 #ifdef PROT_MAX
192 	flags |= PROT_MAX(PROT_READ | PROT_WRITE | PROT_EXEC);
193 #endif
194 #if defined(HAVE_JIT) && (defined(__linux__) || defined(__FreeBSD__)) && (defined(__x86_64__) || defined (__aarch64__)) && !defined(__SANITIZE_ADDRESS__)
195 	void *hint;
196 	if (JIT_G(enabled) && JIT_G(buffer_size)
197 			&& zend_jit_check_support() == SUCCESS) {
198 		hint = find_prefered_mmap_base(requested_size);
199 	} else {
200 		/* Do not use a hint if JIT is not enabled, as this profits only JIT and
201 		 * this is potentially unsafe when the only suitable candidate is just
202 		 * after the heap (e.g. in non-PIE builds) (GH-13775). */
203 		hint = MAP_FAILED;
204 	}
205 	if (hint != MAP_FAILED) {
206 # ifdef MAP_HUGETLB
207 		size_t huge_page_size = 2 * 1024 * 1024;
208 		if (requested_size >= huge_page_size && requested_size % huge_page_size == 0) {
209 			p = mmap(hint, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB|MAP_FIXED, -1, 0);
210 			if (p != MAP_FAILED) {
211 				goto success;
212 			}
213 		}
214 #endif
215 		p = mmap(hint, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
216 		if (p != MAP_FAILED) {
217 			goto success;
218 		}
219 	}
220 #endif
221 #ifdef MAP_HUGETLB
222 	size_t huge_page_size = 2 * 1024 * 1024;
223 
224 	/* Try to allocate huge pages first to reduce dTLB misses.
225 	 * OSes has to be configured properly
226 	 * on Linux
227 	 * (e.g. https://wiki.debian.org/Hugepages#Enabling_HugeTlbPage)
228 	 * You may verify huge page usage with the following command:
229 	 * `grep "Huge" /proc/meminfo`
230 	 * on FreeBSD
231 	 * sysctl vm.pmap.pg_ps_enabled entry
232 	 * (boot time config only, but enabled by default on most arches).
233 	 */
234 	if (requested_size >= huge_page_size && requested_size % huge_page_size == 0) {
235 # if defined(__x86_64__) && defined(MAP_32BIT)
236 		/* to got HUGE PAGES in low 32-bit address we have to reserve address
237 		   space and then remap it using MAP_HUGETLB */
238 
239 		p = mmap(NULL, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, fd, 0);
240 		if (p != MAP_FAILED) {
241 			munmap(p, requested_size);
242 			p = (void*)(ZEND_MM_ALIGNED_SIZE_EX((ptrdiff_t)p, huge_page_size));
243 			p = mmap(p, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT|MAP_HUGETLB|MAP_FIXED, -1, 0);
244 			if (p != MAP_FAILED) {
245 				goto success;
246 			} else {
247 				p = mmap(NULL, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, fd, 0);
248 				if (p != MAP_FAILED) {
249 					goto success;
250 				}
251 			}
252 		}
253 # endif
254 		p = mmap(0, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_HUGETLB, fd, 0);
255 		if (p != MAP_FAILED) {
256 			goto success;
257 		}
258 	}
259 #elif defined(PREFER_MAP_32BIT) && defined(__x86_64__) && defined(MAP_32BIT)
260 	p = mmap(NULL, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS|MAP_32BIT, fd, 0);
261 	if (p != MAP_FAILED) {
262 		goto success;
263 	}
264 #endif
265 
266 	p = mmap(0, requested_size, flags, MAP_SHARED|MAP_ANONYMOUS, fd, 0);
267 	if (p == MAP_FAILED) {
268 		*error_in = "mmap";
269 		return ALLOC_FAILURE;
270 	}
271 
272 success: ZEND_ATTRIBUTE_UNUSED;
273 	*shared_segments_count = 1;
274 	*shared_segments_p = (zend_shared_segment **) calloc(1, sizeof(zend_shared_segment) + sizeof(void *));
275 	if (!*shared_segments_p) {
276 		munmap(p, requested_size);
277 		*error_in = "calloc";
278 		return ALLOC_FAILURE;
279 	}
280 	shared_segment = (zend_shared_segment *)((char *)(*shared_segments_p) + sizeof(void *));
281 	(*shared_segments_p)[0] = shared_segment;
282 
283 	shared_segment->p = p;
284 	shared_segment->pos = 0;
285 	shared_segment->size = requested_size;
286 
287 	return ALLOC_SUCCESS;
288 }
289 
detach_segment(zend_shared_segment * shared_segment)290 static int detach_segment(zend_shared_segment *shared_segment)
291 {
292 	munmap(shared_segment->p, shared_segment->size);
293 	return 0;
294 }
295 
segment_type_size(void)296 static size_t segment_type_size(void)
297 {
298 	return sizeof(zend_shared_segment);
299 }
300 
301 zend_shared_memory_handlers zend_alloc_mmap_handlers = {
302 	create_segments,
303 	detach_segment,
304 	segment_type_size
305 };
306 
307 #endif /* USE_MMAP */
308