xref: /php-src/ext/mbstring/php_unicode.c (revision 67253987)
1 /*
2    +----------------------------------------------------------------------+
3    | Copyright (c) The PHP Group                                          |
4    +----------------------------------------------------------------------+
5    | This source file is subject to version 3.01 of the PHP license,      |
6    | that is bundled with this package in the file LICENSE, and is        |
7    | available through the world-wide-web at the following url:           |
8    | https://www.php.net/license/3_01.txt                                 |
9    | If you did not receive a copy of the PHP license and are unable to   |
10    | obtain it through the world-wide-web, please send a note to          |
11    | license@php.net so we can mail you a copy immediately.               |
12    +----------------------------------------------------------------------+
13    | Author: Wez Furlong (wez@thebrainroom.com)                           |
14    +----------------------------------------------------------------------+
15 
16 	Based on code from ucdata-2.5, which has the following Copyright:
17 
18 	Copyright 2001 Computing Research Labs, New Mexico State University
19 
20 	Permission is hereby granted, free of charge, to any person obtaining a
21 	copy of this software and associated documentation files (the "Software"),
22 	to deal in the Software without restriction, including without limitation
23 	the rights to use, copy, modify, merge, publish, distribute, sublicense,
24 	and/or sell copies of the Software, and to permit persons to whom the
25 	Software is furnished to do so, subject to the following conditions:
26 
27 	The above copyright notice and this permission notice shall be included in
28 	all copies or substantial portions of the Software.
29 */
30 
31 #include "php.h"
32 
33 /* include case folding data generated from the official UnicodeData.txt file */
34 #include "mbstring.h"
35 #include "php_unicode.h"
36 #include "unicode_data.h"
37 
38 extern const mbfl_encoding mbfl_encoding_8859_9;
39 
ZEND_EXTERN_MODULE_GLOBALS(mbstring)40 ZEND_EXTERN_MODULE_GLOBALS(mbstring)
41 
42 static bool prop_lookup(unsigned long code, unsigned long n)
43 {
44 	long l = _ucprop_offsets[n];
45 	long r = _ucprop_offsets[n + 1] - 1;
46 	while (l <= r) {
47 		/*
48 		 * Determine a "mid" point and adjust to make sure the mid point is at
49 		 * the beginning of a range pair.
50 		 */
51 		long m = (l + r) >> 1;
52 		m -= (m & 1);
53 		if (code > _ucprop_ranges[m + 1])
54 			l = m + 2;
55 		else if (code < _ucprop_ranges[m])
56 			r = m - 2;
57 		else
58 			return true;
59 	}
60 	return false;
61 
62 }
63 
php_unicode_is_prop1(unsigned long code,int prop)64 MBSTRING_API bool php_unicode_is_prop1(unsigned long code, int prop)
65 {
66 	return prop_lookup(code, prop);
67 }
68 
php_unicode_is_prop(unsigned long code,...)69 MBSTRING_API bool php_unicode_is_prop(unsigned long code, ...)
70 {
71 	bool result = false;
72 	va_list va;
73 	va_start(va, code);
74 
75 	while (1) {
76 		int prop = va_arg(va, int);
77 		if (prop < 0) {
78 			break;
79 		}
80 
81 		if (prop_lookup(code, prop)) {
82 			result = true;
83 			break;
84 		}
85 	}
86 
87 	va_end(va);
88 	return result;
89 }
90 
mph_hash(unsigned d,unsigned x)91 static inline unsigned mph_hash(unsigned d, unsigned x) {
92 	x ^= d;
93 	x = ((x >> 16) ^ x) * 0x45d9f3b;
94 	return x;
95 }
96 
97 #define CODE_NOT_FOUND ((unsigned) -1)
98 
mph_lookup(unsigned code,const short * g_table,unsigned g_table_size,const unsigned * table,unsigned table_size)99 static inline unsigned mph_lookup(
100 		unsigned code,
101 		const short *g_table, unsigned g_table_size,
102 		const unsigned *table, unsigned table_size)
103 {
104 	short g = g_table[mph_hash(0, code) % g_table_size];
105 
106 	unsigned idx;
107 	if (g <= 0) {
108 		idx = -g;
109 	} else {
110 		idx = mph_hash(g, code) % table_size;
111 	}
112 
113 	if (table[2*idx] == code) {
114 		return table[2*idx + 1];
115 	}
116 	return CODE_NOT_FOUND;
117 }
118 
119 #define CASE_LOOKUP(code, type) \
120 	mph_lookup(code, _uccase_##type##_g, _uccase_##type##_g_size, \
121 			_uccase_##type##_table, _uccase_##type##_table_size)
122 
php_unicode_toupper_raw(unsigned code,const mbfl_encoding * enc)123 static unsigned php_unicode_toupper_raw(unsigned code, const mbfl_encoding *enc)
124 {
125 	/* After the ASCII characters, the first codepoint with an uppercase version
126 	 * is 0xB5 (MICRO SIGN) */
127 	if (code < 0xB5) {
128 		/* Fast path for ASCII */
129 		if (code >= 0x61 && code <= 0x7A) {
130 			if (UNEXPECTED(enc == &mbfl_encoding_8859_9 && code == 0x69)) {
131 				return 0x130;
132 			}
133 			return code - 0x20;
134 		}
135 		return code;
136 	} else {
137 		unsigned new_code = CASE_LOOKUP(code, upper);
138 		if (new_code != CODE_NOT_FOUND) {
139 			return new_code;
140 		}
141 		return code;
142 	}
143 }
144 
php_unicode_tolower_raw(unsigned code,const mbfl_encoding * enc)145 static unsigned php_unicode_tolower_raw(unsigned code, const mbfl_encoding *enc)
146 {
147 	/* After the ASCII characters, the first codepoint with a lowercase version
148 	 * is 0xC0 (LATIN CAPITAL LETTER A WITH GRAVE) */
149 	if (code < 0xC0) {
150 		/* Fast path for ASCII */
151 		if (code >= 0x41 && code <= 0x5A) {
152 			if (UNEXPECTED(enc == &mbfl_encoding_8859_9 && code == 0x0049L)) {
153 				return 0x0131L;
154 			}
155 			return code + 0x20;
156 		}
157 		return code;
158 	} else {
159 		unsigned new_code = CASE_LOOKUP(code, lower);
160 		if (new_code != CODE_NOT_FOUND) {
161 			if (UNEXPECTED(enc == &mbfl_encoding_8859_9 && code == 0x130)) {
162 				return 0x69;
163 			}
164 			return new_code;
165 		}
166 		return code;
167 	}
168 }
169 
php_unicode_totitle_raw(unsigned code,const mbfl_encoding * enc)170 static unsigned php_unicode_totitle_raw(unsigned code, const mbfl_encoding *enc)
171 {
172 	unsigned new_code = CASE_LOOKUP(code, title);
173 	if (new_code != CODE_NOT_FOUND) {
174 		return new_code;
175 	}
176 
177 	/* No dedicated title-case variant, use to-upper instead */
178 	return php_unicode_toupper_raw(code, enc);
179 }
180 
php_unicode_tofold_raw(unsigned code,const mbfl_encoding * enc)181 static unsigned php_unicode_tofold_raw(unsigned code, const mbfl_encoding *enc)
182 {
183 	if (code < 0x80) {
184 		/* Fast path for ASCII */
185 		if (code >= 0x41 && code <= 0x5A) {
186 			if (UNEXPECTED(enc == &mbfl_encoding_8859_9 && code == 0x49)) {
187 				return 0x131;
188 			}
189 			return code + 0x20;
190 		}
191 		return code;
192 	} else {
193 		unsigned new_code = CASE_LOOKUP(code, fold);
194 		if (new_code != CODE_NOT_FOUND) {
195 			if (UNEXPECTED(enc == &mbfl_encoding_8859_9 && code == 0x130)) {
196 				return 0x69;
197 			}
198 			return new_code;
199 		}
200 		return code;
201 	}
202 }
203 
php_unicode_tolower_simple(unsigned code,const mbfl_encoding * enc)204 static inline unsigned php_unicode_tolower_simple(unsigned code, const mbfl_encoding *enc) {
205 	code = php_unicode_tolower_raw(code, enc);
206 	if (UNEXPECTED(code > 0xffffff)) {
207 		return _uccase_extra_table[code & 0xffffff];
208 	}
209 	return code;
210 }
php_unicode_toupper_simple(unsigned code,const mbfl_encoding * enc)211 static inline unsigned php_unicode_toupper_simple(unsigned code, const mbfl_encoding *enc) {
212 	code = php_unicode_toupper_raw(code, enc);
213 	if (UNEXPECTED(code > 0xffffff)) {
214 		return _uccase_extra_table[code & 0xffffff];
215 	}
216 	return code;
217 }
php_unicode_totitle_simple(unsigned code,const mbfl_encoding * enc)218 static inline unsigned php_unicode_totitle_simple(unsigned code, const mbfl_encoding *enc) {
219 	code = php_unicode_totitle_raw(code, enc);
220 	if (UNEXPECTED(code > 0xffffff)) {
221 		return _uccase_extra_table[code & 0xffffff];
222 	}
223 	return code;
224 }
php_unicode_tofold_simple(unsigned code,const mbfl_encoding * enc)225 static inline unsigned php_unicode_tofold_simple(unsigned code, const mbfl_encoding *enc) {
226 	code = php_unicode_tofold_raw(code, enc);
227 	if (UNEXPECTED(code > 0xffffff)) {
228 		return _uccase_extra_table[code & 0xffffff];
229 	}
230 	return code;
231 }
232 
emit_special_casing_sequence(uint32_t w,uint32_t * out)233 static uint32_t *emit_special_casing_sequence(uint32_t w, uint32_t *out)
234 {
235 	unsigned int len = w >> 24;
236 	const unsigned int *p = &_uccase_extra_table[w & 0xFFFFFF];
237 	while (len--) {
238 		*out++ = *++p;
239 	}
240 	return out;
241 }
242 
243 /* Used when determining whether special casing rules should be applied to Greek letter sigma */
scan_ahead_for_cased_letter(unsigned char * in,size_t in_len,unsigned int state,const mbfl_encoding * encoding)244 static bool scan_ahead_for_cased_letter(unsigned char *in, size_t in_len, unsigned int state, const mbfl_encoding *encoding)
245 {
246 	uint32_t wchar_buf[64];
247 
248 	while (in_len) {
249 		size_t out_len = encoding->to_wchar(&in, &in_len, wchar_buf, 64, &state);
250 		ZEND_ASSERT(out_len <= 64);
251 		for (unsigned int i = 0; i < out_len; i++) {
252 			uint32_t w = wchar_buf[i];
253 			if (php_unicode_is_cased(w)) {
254 				return true;
255 			}
256 			if (!php_unicode_is_case_ignorable(w)) {
257 				return false;
258 			}
259 		}
260 	}
261 
262 	return false;
263 }
264 
265 /* Used when determining whether special casing rules should be applied to Greek letter sigma */
scan_back_for_cased_letter(uint32_t * begin,uint32_t * end)266 static bool scan_back_for_cased_letter(uint32_t *begin, uint32_t *end)
267 {
268 	if (end != NULL) {
269 		while (--end >= begin) {
270 			uint32_t w = *end;
271 			if (php_unicode_is_cased(w)) {
272 				return true;
273 			}
274 			if (!php_unicode_is_case_ignorable(w)) {
275 				return false;
276 			}
277 		}
278 	}
279 	return false;
280 }
281 
php_unicode_convert_case(php_case_mode case_mode,const char * srcstr,size_t in_len,const mbfl_encoding * src_encoding,const mbfl_encoding * dst_encoding,int illegal_mode,uint32_t illegal_substchar)282 MBSTRING_API zend_string *php_unicode_convert_case(php_case_mode case_mode, const char *srcstr, size_t in_len, const mbfl_encoding *src_encoding, const mbfl_encoding *dst_encoding, int illegal_mode, uint32_t illegal_substchar)
283 {
284 	/* A Unicode codepoint can expand out to up to 3 codepoints when uppercased, lowercased, or title cased
285 	 * See http://www.unicode.org/Public/UNIDATA/SpecialCasing.txt */
286 	uint32_t wchar_buf[64], converted_buf[192];
287 	unsigned int state = 0, title_mode = 0;
288 	unsigned char *in = (unsigned char*)srcstr;
289 	/* In rare cases, we need to scan backwards through the previously converted codepoints to see
290 	 * if special conversion rules should be used for the Greek letter sigma */
291 	uint32_t *converted_end = NULL;
292 
293 	mb_convert_buf buf;
294 	mb_convert_buf_init(&buf, in_len + 1, illegal_substchar, illegal_mode);
295 
296 	while (in_len) {
297 		size_t out_len = src_encoding->to_wchar(&in, &in_len, wchar_buf, 64, &state);
298 		ZEND_ASSERT(out_len <= 64);
299 		uint32_t *p = converted_buf;
300 
301 		/* In all cases, handle invalid characters early, as we assign special meaning to codepoints > 0xFFFFFF */
302 		switch (case_mode) {
303 		case PHP_UNICODE_CASE_UPPER_SIMPLE:
304 			for (size_t i = 0; i < out_len; i++) {
305 				uint32_t w = wchar_buf[i];
306 				*p++ = (UNEXPECTED(w > 0xFFFFFF)) ? w : php_unicode_toupper_simple(w, src_encoding);
307 			}
308 			break;
309 
310 		case PHP_UNICODE_CASE_LOWER_SIMPLE:
311 			for (size_t i = 0; i < out_len; i++) {
312 				uint32_t w = wchar_buf[i];
313 				*p++ = (UNEXPECTED(w > 0xFFFFFF)) ? w : php_unicode_tolower_simple(w, src_encoding);
314 			}
315 			break;
316 
317 		case PHP_UNICODE_CASE_FOLD_SIMPLE:
318 			for (size_t i = 0; i < out_len; i++) {
319 				uint32_t w = wchar_buf[i];
320 				*p++ = (UNEXPECTED(w > 0xFFFFFF)) ? w : php_unicode_tofold_simple(w, src_encoding);
321 			}
322 			break;
323 
324 		case PHP_UNICODE_CASE_TITLE_SIMPLE:
325 			for (size_t i = 0; i < out_len; i++) {
326 				uint32_t w = wchar_buf[i];
327 				if (UNEXPECTED(w > 0xFFFFFF)) {
328 					*p++ = w;
329 					continue;
330 				}
331 				*p++ = title_mode ? php_unicode_tolower_simple(w, src_encoding) : php_unicode_totitle_simple(w, src_encoding);
332 				if (!php_unicode_is_case_ignorable(w)) {
333 					title_mode = php_unicode_is_cased(w);
334 				}
335 			}
336 			break;
337 
338 		case PHP_UNICODE_CASE_UPPER:
339 			for (size_t i = 0; i < out_len; i++) {
340 				uint32_t w = wchar_buf[i];
341 				if (UNEXPECTED(w > 0xFFFFFF)) {
342 					*p++ = w;
343 					continue;
344 				}
345 				w = php_unicode_toupper_raw(w, src_encoding);
346 				if (UNEXPECTED(w > 0xFFFFFF)) {
347 					p = emit_special_casing_sequence(w, p);
348 				} else {
349 					*p++ = w;
350 				}
351 			}
352 			break;
353 
354 		case PHP_UNICODE_CASE_LOWER:
355 			for (size_t i = 0; i < out_len; i++) {
356 				uint32_t w = wchar_buf[i];
357 				if (UNEXPECTED(w > 0xFFFFFF)) {
358 					*p++ = w;
359 					continue;
360 				}
361 				if (w == 0x3A3) {
362 					/* For Greek capital letter sigma, there is a special casing rule;
363 					 * if it is the last letter in a word, it should be downcased to U+03C2
364 					 * (GREEK SMALL LETTER FINAL SIGMA)
365 					 * Specifically, we need to check if this codepoint is preceded by any
366 					 * number of case-ignorable codepoints, preceded by a cased letter, AND
367 					 * is NOT followed by any number of case-ignorable codepoints followed
368 					 * by a cased letter.
369 					 * Ref: http://www.unicode.org/reports/tr21/tr21-5.html
370 					 * Ref: https://unicode.org/Public/UNIDATA/SpecialCasing.txt
371 					 *
372 					 * While the special casing rules say we should scan backwards through "any number"
373 					 * of case-ignorable codepoints, that is a great implementation burden
374 					 * It would basically mean we need to keep all the codepoints in a big buffer
375 					 * during this conversion operation, but we don't want to do that (to reduce the
376 					 * amount of temporary scratch memory used)
377 					 * Hence, we only scan back through the codepoints in wchar_buf, and if we hit the
378 					 * beginning of the buffer, whatever codepoints have not yet been overwritten in
379 					 * the latter part of converted_buf */
380 					int j = i - 1;
381 					while (j >= 0 && php_unicode_is_case_ignorable(wchar_buf[j])) {
382 						j--;
383 					}
384 					if (j >= 0 ? php_unicode_is_cased(wchar_buf[j]) : scan_back_for_cased_letter(p, converted_end)) {
385 						/* Now scan ahead to look for a cased letter */
386 						j = i + 1;
387 						while (j < out_len && php_unicode_is_case_ignorable(wchar_buf[j])) {
388 							j++;
389 						}
390 						/* If we hit the end of wchar_buf, convert more of the input string into
391 						 * codepoints and continue scanning */
392 						if (j >= out_len ? !scan_ahead_for_cased_letter(in, in_len, state, src_encoding) : !php_unicode_is_cased(wchar_buf[j])) {
393 							*p++ = 0x3C2;
394 							continue;
395 						}
396 					}
397 				}
398 				w = php_unicode_tolower_raw(w, src_encoding);
399 				if (UNEXPECTED(w > 0xFFFFFF)) {
400 					p = emit_special_casing_sequence(w, p);
401 				} else {
402 					*p++ = w;
403 				}
404 			}
405 			break;
406 
407 		case PHP_UNICODE_CASE_FOLD:
408 			for (size_t i = 0; i < out_len; i++) {
409 				uint32_t w = wchar_buf[i];
410 				if (UNEXPECTED(w > 0xFFFFFF)) {
411 					*p++ = w;
412 					continue;
413 				}
414 				w = php_unicode_tofold_raw(w, src_encoding);
415 				if (UNEXPECTED(w > 0xFFFFFF)) {
416 					p = emit_special_casing_sequence(w, p);
417 				} else {
418 					*p++ = w;
419 				}
420 			}
421 			break;
422 
423 		case PHP_UNICODE_CASE_TITLE:
424 			for (size_t i = 0; i < out_len; i++) {
425 				uint32_t w = wchar_buf[i];
426 				if (UNEXPECTED(w > 0xFFFFFF)) {
427 					*p++ = w;
428 					continue;
429 				}
430 				uint32_t w2;
431 				if (title_mode) {
432 					if (w == 0x3A3) {
433 						int j = i - 1;
434 						while (j >= 0 && php_unicode_is_case_ignorable(wchar_buf[j])) {
435 							j--;
436 						}
437 						if (j >= 0 ? php_unicode_is_cased(wchar_buf[j]) : scan_back_for_cased_letter(p, converted_end)) {
438 							j = i + 1;
439 							while (j < out_len && php_unicode_is_case_ignorable(wchar_buf[j])) {
440 								j++;
441 							}
442 							if (j >= out_len ? !scan_ahead_for_cased_letter(in, in_len, state, src_encoding) : !php_unicode_is_cased(wchar_buf[j])) {
443 								*p++ = 0x3C2;
444 								goto set_title_mode;
445 							}
446 						}
447 					}
448 					w2 = php_unicode_tolower_raw(w, src_encoding);
449 				} else {
450 					w2 = php_unicode_totitle_raw(w, src_encoding);
451 				}
452 				if (UNEXPECTED(w2 > 0xFFFFFF)) {
453 					p = emit_special_casing_sequence(w2, p);
454 				} else {
455 					*p++ = w2;
456 				}
457 set_title_mode:
458 				if (!php_unicode_is_case_ignorable(w)) {
459 					title_mode = php_unicode_is_cased(w);
460 				}
461 			}
462 			break;
463 
464 			EMPTY_SWITCH_DEFAULT_CASE()
465 		}
466 
467 		converted_end = p;
468 		ZEND_ASSERT(p - converted_buf <= 192);
469 		dst_encoding->from_wchar(converted_buf, p - converted_buf, &buf, !in_len);
470 	}
471 
472 	return mb_convert_buf_result(&buf, dst_encoding);
473 }
474