xref: /PHP-7.3/ext/mbstring/oniguruma/src/utf8.c (revision 1979c5d1)
1 /**********************************************************************
2   utf8.c -  Oniguruma (regular expression library)
3 **********************************************************************/
4 /*-
5  * Copyright (c) 2002-2019  K.Kosako
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include "regenc.h"
31 
32 /* U+0000 - U+10FFFF */
33 #define USE_RFC3629_RANGE
34 
35 /* #define USE_INVALID_CODE_SCHEME */
36 
37 #ifdef USE_INVALID_CODE_SCHEME
38 /* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
39 #define INVALID_CODE_FE   0xfffffffe
40 #define INVALID_CODE_FF   0xffffffff
41 #define VALID_CODE_LIMIT  0x7fffffff
42 #endif
43 
44 #define utf8_islead(c)     ((UChar )((c) & 0xc0) != 0x80)
45 #define utf8_istail(c)     ((UChar )((c) & 0xc0) == 0x80)
46 
47 static const int EncLen_UTF8[] = {
48   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
49   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
50   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
51   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
52   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
53   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
54   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
55   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
56   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
57   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
58   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
59   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
60   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
61   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
62   3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
63 #ifdef USE_RFC3629_RANGE
64   4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1
65 #else
66   4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1
67 #endif
68 };
69 
70 static int
mbc_enc_len(const UChar * p)71 mbc_enc_len(const UChar* p)
72 {
73   return EncLen_UTF8[*p];
74 }
75 
76 static int
is_valid_mbc_string(const UChar * p,const UChar * end)77 is_valid_mbc_string(const UChar* p, const UChar* end)
78 {
79   int i, len;
80 
81   while (p < end) {
82     if (! utf8_islead(*p))
83       return FALSE;
84 
85     len = mbc_enc_len(p++);
86     if (len > 1) {
87       for (i = 1; i < len; i++) {
88         if (p == end)
89           return FALSE;
90 
91         if (! utf8_istail(*p++))
92           return FALSE;
93       }
94     }
95   }
96 
97   return TRUE;
98 }
99 
100 static OnigCodePoint
mbc_to_code(const UChar * p,const UChar * end)101 mbc_to_code(const UChar* p, const UChar* end)
102 {
103   int c, len;
104   OnigCodePoint n;
105 
106   len = mbc_enc_len(p);
107   if (len > (int )(end - p)) len = (int )(end - p);
108 
109   c = *p++;
110   if (len > 1) {
111     len--;
112     n = c & ((1 << (6 - len)) - 1);
113     while (len--) {
114       c = *p++;
115       n = (n << 6) | (c & ((1 << 6) - 1));
116     }
117     return n;
118   }
119   else {
120 #ifdef USE_INVALID_CODE_SCHEME
121     if (c > 0xfd) {
122       return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
123     }
124 #endif
125     return (OnigCodePoint )c;
126   }
127 }
128 
129 static int
code_to_mbclen(OnigCodePoint code)130 code_to_mbclen(OnigCodePoint code)
131 {
132   if      ((code & 0xffffff80) == 0) return 1;
133   else if ((code & 0xfffff800) == 0) return 2;
134   else if ((code & 0xffff0000) == 0) return 3;
135   else if ((code & 0xffe00000) == 0) return 4;
136 #ifndef USE_RFC3629_RANGE
137   else if ((code & 0xfc000000) == 0) return 5;
138   else if ((code & 0x80000000) == 0) return 6;
139 #endif
140 #ifdef USE_INVALID_CODE_SCHEME
141   else if (code == INVALID_CODE_FE) return 1;
142   else if (code == INVALID_CODE_FF) return 1;
143 #endif
144   else
145     return ONIGERR_INVALID_CODE_POINT_VALUE;
146 }
147 
148 static int
code_to_mbc(OnigCodePoint code,UChar * buf)149 code_to_mbc(OnigCodePoint code, UChar *buf)
150 {
151 #define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
152 #define UTF8_TRAIL0(code)        (UChar )(((code) & 0x3f) | 0x80)
153 
154   if ((code & 0xffffff80) == 0) {
155     *buf = (UChar )code;
156     return 1;
157   }
158   else {
159     UChar *p = buf;
160 
161     if ((code & 0xfffff800) == 0) {
162       *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
163     }
164     else if ((code & 0xffff0000) == 0) {
165       *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
166       *p++ = UTF8_TRAILS(code, 6);
167     }
168     else if ((code & 0xffe00000) == 0) {
169       *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
170       *p++ = UTF8_TRAILS(code, 12);
171       *p++ = UTF8_TRAILS(code,  6);
172     }
173 #ifndef USE_RFC3629_RANGE
174     else if ((code & 0xfc000000) == 0) {
175       *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
176       *p++ = UTF8_TRAILS(code, 18);
177       *p++ = UTF8_TRAILS(code, 12);
178       *p++ = UTF8_TRAILS(code,  6);
179     }
180     else if ((code & 0x80000000) == 0) {
181       *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
182       *p++ = UTF8_TRAILS(code, 24);
183       *p++ = UTF8_TRAILS(code, 18);
184       *p++ = UTF8_TRAILS(code, 12);
185       *p++ = UTF8_TRAILS(code,  6);
186     }
187 #endif
188 #ifdef USE_INVALID_CODE_SCHEME
189     else if (code == INVALID_CODE_FE) {
190       *p = 0xfe;
191       return 1;
192     }
193     else if (code == INVALID_CODE_FF) {
194       *p = 0xff;
195       return 1;
196     }
197 #endif
198     else {
199       return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
200     }
201 
202     *p++ = UTF8_TRAIL0(code);
203     return (int )(p - buf);
204   }
205 }
206 
207 static int
mbc_case_fold(OnigCaseFoldType flag,const UChar ** pp,const UChar * end,UChar * fold)208 mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
209               const UChar* end, UChar* fold)
210 {
211   const UChar* p = *pp;
212 
213   if (ONIGENC_IS_MBC_ASCII(p)) {
214 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
215     if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
216       if (*p == 0x49) {
217         *fold++ = 0xc4;
218         *fold   = 0xb1;
219         (*pp)++;
220         return 2;
221       }
222     }
223 #endif
224 
225     *fold = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
226     (*pp)++;
227     return 1; /* return byte length of converted char to lower */
228   }
229   else {
230     return onigenc_unicode_mbc_case_fold(ONIG_ENCODING_UTF8, flag,
231                                          pp, end, fold);
232   }
233 }
234 
235 static int
get_ctype_code_range(OnigCtype ctype,OnigCodePoint * sb_out,const OnigCodePoint * ranges[])236 get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
237                      const OnigCodePoint* ranges[])
238 {
239   *sb_out = 0x80;
240   return onigenc_unicode_ctype_code_range(ctype, ranges);
241 }
242 
243 
244 static UChar*
left_adjust_char_head(const UChar * start,const UChar * s)245 left_adjust_char_head(const UChar* start, const UChar* s)
246 {
247   const UChar *p;
248 
249   if (s <= start) return (UChar* )s;
250   p = s;
251 
252   while (!utf8_islead(*p) && p > start) p--;
253   return (UChar* )p;
254 }
255 
256 static int
get_case_fold_codes_by_str(OnigCaseFoldType flag,const OnigUChar * p,const OnigUChar * end,OnigCaseFoldCodeItem items[])257 get_case_fold_codes_by_str(OnigCaseFoldType flag,
258     const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[])
259 {
260   return onigenc_unicode_get_case_fold_codes_by_str(ONIG_ENCODING_UTF8,
261                                                     flag, p, end, items);
262 }
263 
264 OnigEncodingType OnigEncodingUTF8 = {
265   mbc_enc_len,
266   "UTF-8",     /* name */
267 #ifdef USE_RFC3629_RANGE
268   4,           /* max enc length */
269 #else
270   6,
271 #endif
272   1,           /* min enc length */
273   onigenc_is_mbc_newline_0x0a,
274   mbc_to_code,
275   code_to_mbclen,
276   code_to_mbc,
277   mbc_case_fold,
278   onigenc_unicode_apply_all_case_fold,
279   get_case_fold_codes_by_str,
280   onigenc_unicode_property_name_to_ctype,
281   onigenc_unicode_is_code_ctype,
282   get_ctype_code_range,
283   left_adjust_char_head,
284   onigenc_always_true_is_allowed_reverse_match,
285   NULL, /* init */
286   NULL, /* is_initialized */
287   is_valid_mbc_string,
288   ENC_FLAG_ASCII_COMPATIBLE|ENC_FLAG_UNICODE|ENC_FLAG_SKIP_OFFSET_1_OR_0,
289   0, 0
290 };
291