xref: /PHP-5.6/ext/mbstring/oniguruma/enc/utf8.c (revision d1fbc98f)
1 /**********************************************************************
2   utf8.c -  Oniguruma (regular expression library)
3 **********************************************************************/
4 /*-
5  * Copyright (c) 2002-2007  K.Kosako  <sndgk393 AT ybb DOT ne DOT jp>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include "regenc.h"
31 
32 #define USE_INVALID_CODE_SCHEME
33 
34 #ifdef USE_INVALID_CODE_SCHEME
35 /* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
36 #define INVALID_CODE_FE   0xfffffffe
37 #define INVALID_CODE_FF   0xffffffff
38 #define VALID_CODE_LIMIT  0x7fffffff
39 #endif
40 
41 #define utf8_islead(c)     ((UChar )((c) & 0xc0) != 0x80)
42 
43 static const int EncLen_UTF8[] = {
44   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
45   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
46   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
47   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
48   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
49   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
50   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
51   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
52   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
53   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
54   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
55   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
56   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
57   2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
58   3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
59   4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1
60 };
61 
62 static int
mbc_enc_len(const UChar * p)63 mbc_enc_len(const UChar* p)
64 {
65   return EncLen_UTF8[*p];
66 }
67 
68 static int
is_mbc_newline(const UChar * p,const UChar * end)69 is_mbc_newline(const UChar* p, const UChar* end)
70 {
71   if (p < end) {
72     if (*p == 0x0a) return 1;
73 
74 #ifdef USE_UNICODE_ALL_LINE_TERMINATORS
75 #ifndef USE_CRNL_AS_LINE_TERMINATOR
76     if (*p == 0x0d) return 1;
77 #endif
78     if (p + 1 < end) {
79       if (*(p+1) == 0x85 && *p == 0xc2) /* U+0085 */
80 	return 1;
81       if (p + 2 < end) {
82 	if ((*(p+2) == 0xa8 || *(p+2) == 0xa9)
83 	    && *(p+1) == 0x80 && *p == 0xe2)  /* U+2028, U+2029 */
84 	  return 1;
85       }
86     }
87 #endif
88   }
89 
90   return 0;
91 }
92 
93 static OnigCodePoint
mbc_to_code(const UChar * p,const UChar * end)94 mbc_to_code(const UChar* p, const UChar* end)
95 {
96   int c, len;
97   OnigCodePoint n;
98 
99   len = mbc_enc_len(p);
100   if (len > end - p) len = end - p;
101 
102   c = *p++;
103   if (len > 1) {
104     len--;
105     n = c & ((1 << (6 - len)) - 1);
106     while (len--) {
107       c = *p++;
108       n = (n << 6) | (c & ((1 << 6) - 1));
109     }
110     return n;
111   }
112   else {
113 #ifdef USE_INVALID_CODE_SCHEME
114     if (c > 0xfd) {
115       return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
116     }
117 #endif
118     return (OnigCodePoint )c;
119   }
120 }
121 
122 static int
code_to_mbclen(OnigCodePoint code)123 code_to_mbclen(OnigCodePoint code)
124 {
125   if      ((code & 0xffffff80) == 0) return 1;
126   else if ((code & 0xfffff800) == 0) return 2;
127   else if ((code & 0xffff0000) == 0) return 3;
128   else if ((code & 0xffe00000) == 0) return 4;
129   else if ((code & 0xfc000000) == 0) return 5;
130   else if ((code & 0x80000000) == 0) return 6;
131 #ifdef USE_INVALID_CODE_SCHEME
132   else if (code == INVALID_CODE_FE) return 1;
133   else if (code == INVALID_CODE_FF) return 1;
134 #endif
135   else
136     return ONIGERR_INVALID_CODE_POINT_VALUE;
137 }
138 
139 static int
code_to_mbc(OnigCodePoint code,UChar * buf)140 code_to_mbc(OnigCodePoint code, UChar *buf)
141 {
142 #define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
143 #define UTF8_TRAIL0(code)        (UChar )(((code) & 0x3f) | 0x80)
144 
145   if ((code & 0xffffff80) == 0) {
146     *buf = (UChar )code;
147     return 1;
148   }
149   else {
150     UChar *p = buf;
151 
152     if ((code & 0xfffff800) == 0) {
153       *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
154     }
155     else if ((code & 0xffff0000) == 0) {
156       *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
157       *p++ = UTF8_TRAILS(code, 6);
158     }
159     else if ((code & 0xffe00000) == 0) {
160       *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
161       *p++ = UTF8_TRAILS(code, 12);
162       *p++ = UTF8_TRAILS(code,  6);
163     }
164     else if ((code & 0xfc000000) == 0) {
165       *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
166       *p++ = UTF8_TRAILS(code, 18);
167       *p++ = UTF8_TRAILS(code, 12);
168       *p++ = UTF8_TRAILS(code,  6);
169     }
170     else if ((code & 0x80000000) == 0) {
171       *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
172       *p++ = UTF8_TRAILS(code, 24);
173       *p++ = UTF8_TRAILS(code, 18);
174       *p++ = UTF8_TRAILS(code, 12);
175       *p++ = UTF8_TRAILS(code,  6);
176     }
177 #ifdef USE_INVALID_CODE_SCHEME
178     else if (code == INVALID_CODE_FE) {
179       *p = 0xfe;
180       return 1;
181     }
182     else if (code == INVALID_CODE_FF) {
183       *p = 0xff;
184       return 1;
185     }
186 #endif
187     else {
188       return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
189     }
190 
191     *p++ = UTF8_TRAIL0(code);
192     return p - buf;
193   }
194 }
195 
196 static int
mbc_case_fold(OnigCaseFoldType flag,const UChar ** pp,const UChar * end,UChar * fold)197 mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
198 	      const UChar* end, UChar* fold)
199 {
200   const UChar* p = *pp;
201 
202   if (ONIGENC_IS_MBC_ASCII(p)) {
203 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
204     if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
205       if (*p == 0x49) {
206 	*fold++ = 0xc4;
207 	*fold   = 0xb1;
208 	(*pp)++;
209 	return 2;
210       }
211     }
212 #endif
213 
214     *fold = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
215     (*pp)++;
216     return 1; /* return byte length of converted char to lower */
217   }
218   else {
219     return onigenc_unicode_mbc_case_fold(ONIG_ENCODING_UTF8, flag,
220 					 pp, end, fold);
221   }
222 }
223 
224 #if 0
225 static int
226 is_mbc_ambiguous(OnigCaseFoldType flag, const UChar** pp, const UChar* end)
227 {
228   const UChar* p = *pp;
229 
230   if (ONIGENC_IS_MBC_ASCII(p)) {
231     (*pp)++;
232     return ONIGENC_IS_ASCII_CODE_CASE_AMBIG(*p);
233   }
234   else {
235     (*pp) += enclen(ONIG_ENCODING_UTF8, p);
236 
237     if (*p == 0xc3) {
238       int c = *(p + 1);
239       if (c >= 0x80) {
240 	if (c <= (UChar )0x9e) { /* upper */
241 	  if (c == (UChar )0x97) return FALSE;
242 	  return TRUE;
243 	}
244 	else if (c >= (UChar )0xa0 && c <= (UChar )0xbe) { /* lower */
245 	  if (c == (UChar )'\267') return FALSE;
246 	  return TRUE;
247 	}
248         else if (c == (UChar )0x9f &&
249                  (flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) != 0) {
250 	  return TRUE;
251         }
252       }
253     }
254   }
255 
256   return FALSE;
257 }
258 #endif
259 
260 
261 static int
get_ctype_code_range(OnigCtype ctype,OnigCodePoint * sb_out,const OnigCodePoint * ranges[])262 get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
263 		     const OnigCodePoint* ranges[])
264 {
265   *sb_out = 0x80;
266   return onigenc_unicode_ctype_code_range(ctype, ranges);
267 }
268 
269 
270 static UChar*
left_adjust_char_head(const UChar * start,const UChar * s)271 left_adjust_char_head(const UChar* start, const UChar* s)
272 {
273   const UChar *p;
274 
275   if (s <= start) return (UChar* )s;
276   p = s;
277 
278   while (!utf8_islead(*p) && p > start) p--;
279   return (UChar* )p;
280 }
281 
282 static int
get_case_fold_codes_by_str(OnigCaseFoldType flag,const OnigUChar * p,const OnigUChar * end,OnigCaseFoldCodeItem items[])283 get_case_fold_codes_by_str(OnigCaseFoldType flag,
284     const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[])
285 {
286   return onigenc_unicode_get_case_fold_codes_by_str(ONIG_ENCODING_UTF8,
287 						    flag, p, end, items);
288 }
289 
290 OnigEncodingType OnigEncodingUTF8 = {
291   mbc_enc_len,
292   "UTF-8",     /* name */
293   6,           /* max byte length */
294   1,           /* min byte length */
295   is_mbc_newline,
296   mbc_to_code,
297   code_to_mbclen,
298   code_to_mbc,
299   mbc_case_fold,
300   onigenc_unicode_apply_all_case_fold,
301   get_case_fold_codes_by_str,
302   onigenc_unicode_property_name_to_ctype,
303   onigenc_unicode_is_code_ctype,
304   get_ctype_code_range,
305   left_adjust_char_head,
306   onigenc_always_true_is_allowed_reverse_match
307 };
308