1 /**********************************************************************
2 utf8.c - Oniguruma (regular expression library)
3 **********************************************************************/
4 /*-
5 * Copyright (c) 2002-2016 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "regenc.h"
31
32 //#define USE_INVALID_CODE_SCHEME
33
34 #ifdef USE_INVALID_CODE_SCHEME
35 /* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
36 #define INVALID_CODE_FE 0xfffffffe
37 #define INVALID_CODE_FF 0xffffffff
38 #define VALID_CODE_LIMIT 0x7fffffff
39 #endif
40
41 #define utf8_islead(c) ((UChar )((c) & 0xc0) != 0x80)
42 #define utf8_istail(c) ((UChar )((c) & 0xc0) == 0x80)
43
44 static const int EncLen_UTF8[] = {
45 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
46 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
47 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
48 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
49 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
50 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
51 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
52 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
53 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
54 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
55 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
56 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
57 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
58 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
59 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
60 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1
61 };
62
63 static int
mbc_enc_len(const UChar * p)64 mbc_enc_len(const UChar* p)
65 {
66 return EncLen_UTF8[*p];
67 }
68
69 static int
is_valid_mbc_string(const UChar * p,const UChar * end)70 is_valid_mbc_string(const UChar* p, const UChar* end)
71 {
72 int i, len;
73
74 while (p < end) {
75 if (! utf8_islead(*p))
76 return FALSE;
77
78 len = mbc_enc_len(p++);
79 if (len > 1) {
80 for (i = 1; i < len; i++) {
81 if (p == end)
82 return FALSE;
83
84 if (! utf8_istail(*p++))
85 return FALSE;
86 }
87 }
88 }
89
90 return TRUE;
91 }
92
93 static int
is_mbc_newline(const UChar * p,const UChar * end)94 is_mbc_newline(const UChar* p, const UChar* end)
95 {
96 if (p < end) {
97 if (*p == 0x0a) return 1;
98
99 #ifdef USE_UNICODE_ALL_LINE_TERMINATORS
100 #ifndef USE_CRNL_AS_LINE_TERMINATOR
101 if (*p == 0x0d) return 1;
102 #endif
103 if (p + 1 < end) {
104 if (*(p+1) == 0x85 && *p == 0xc2) /* U+0085 */
105 return 1;
106 if (p + 2 < end) {
107 if ((*(p+2) == 0xa8 || *(p+2) == 0xa9)
108 && *(p+1) == 0x80 && *p == 0xe2) /* U+2028, U+2029 */
109 return 1;
110 }
111 }
112 #endif
113 }
114
115 return 0;
116 }
117
118 static OnigCodePoint
mbc_to_code(const UChar * p,const UChar * end)119 mbc_to_code(const UChar* p, const UChar* end)
120 {
121 int c, len;
122 OnigCodePoint n;
123
124 len = mbc_enc_len(p);
125 if (len > end - p) len = end - p;
126
127 c = *p++;
128 if (len > 1) {
129 len--;
130 n = c & ((1 << (6 - len)) - 1);
131 while (len--) {
132 c = *p++;
133 n = (n << 6) | (c & ((1 << 6) - 1));
134 }
135 return n;
136 }
137 else {
138 #ifdef USE_INVALID_CODE_SCHEME
139 if (c > 0xfd) {
140 return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
141 }
142 #endif
143 return (OnigCodePoint )c;
144 }
145 }
146
147 static int
code_to_mbclen(OnigCodePoint code)148 code_to_mbclen(OnigCodePoint code)
149 {
150 if ((code & 0xffffff80) == 0) return 1;
151 else if ((code & 0xfffff800) == 0) return 2;
152 else if ((code & 0xffff0000) == 0) return 3;
153 else if ((code & 0xffe00000) == 0) return 4;
154 else if ((code & 0xfc000000) == 0) return 5;
155 else if ((code & 0x80000000) == 0) return 6;
156 #ifdef USE_INVALID_CODE_SCHEME
157 else if (code == INVALID_CODE_FE) return 1;
158 else if (code == INVALID_CODE_FF) return 1;
159 #endif
160 else
161 return ONIGERR_INVALID_CODE_POINT_VALUE;
162 }
163
164 static int
code_to_mbc(OnigCodePoint code,UChar * buf)165 code_to_mbc(OnigCodePoint code, UChar *buf)
166 {
167 #define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
168 #define UTF8_TRAIL0(code) (UChar )(((code) & 0x3f) | 0x80)
169
170 if ((code & 0xffffff80) == 0) {
171 *buf = (UChar )code;
172 return 1;
173 }
174 else {
175 UChar *p = buf;
176
177 if ((code & 0xfffff800) == 0) {
178 *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
179 }
180 else if ((code & 0xffff0000) == 0) {
181 *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
182 *p++ = UTF8_TRAILS(code, 6);
183 }
184 else if ((code & 0xffe00000) == 0) {
185 *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
186 *p++ = UTF8_TRAILS(code, 12);
187 *p++ = UTF8_TRAILS(code, 6);
188 }
189 else if ((code & 0xfc000000) == 0) {
190 *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
191 *p++ = UTF8_TRAILS(code, 18);
192 *p++ = UTF8_TRAILS(code, 12);
193 *p++ = UTF8_TRAILS(code, 6);
194 }
195 else if ((code & 0x80000000) == 0) {
196 *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
197 *p++ = UTF8_TRAILS(code, 24);
198 *p++ = UTF8_TRAILS(code, 18);
199 *p++ = UTF8_TRAILS(code, 12);
200 *p++ = UTF8_TRAILS(code, 6);
201 }
202 #ifdef USE_INVALID_CODE_SCHEME
203 else if (code == INVALID_CODE_FE) {
204 *p = 0xfe;
205 return 1;
206 }
207 else if (code == INVALID_CODE_FF) {
208 *p = 0xff;
209 return 1;
210 }
211 #endif
212 else {
213 return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
214 }
215
216 *p++ = UTF8_TRAIL0(code);
217 return p - buf;
218 }
219 }
220
221 static int
mbc_case_fold(OnigCaseFoldType flag,const UChar ** pp,const UChar * end,UChar * fold)222 mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
223 const UChar* end, UChar* fold)
224 {
225 const UChar* p = *pp;
226
227 if (ONIGENC_IS_MBC_ASCII(p)) {
228 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
229 if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
230 if (*p == 0x49) {
231 *fold++ = 0xc4;
232 *fold = 0xb1;
233 (*pp)++;
234 return 2;
235 }
236 }
237 #endif
238
239 *fold = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
240 (*pp)++;
241 return 1; /* return byte length of converted char to lower */
242 }
243 else {
244 return onigenc_unicode_mbc_case_fold(ONIG_ENCODING_UTF8, flag,
245 pp, end, fold);
246 }
247 }
248
249 #if 0
250 static int
251 is_mbc_ambiguous(OnigCaseFoldType flag, const UChar** pp, const UChar* end)
252 {
253 const UChar* p = *pp;
254
255 if (ONIGENC_IS_MBC_ASCII(p)) {
256 (*pp)++;
257 return ONIGENC_IS_ASCII_CODE_CASE_AMBIG(*p);
258 }
259 else {
260 (*pp) += enclen(ONIG_ENCODING_UTF8, p);
261
262 if (*p == 0xc3) {
263 int c = *(p + 1);
264 if (c >= 0x80) {
265 if (c <= (UChar )0x9e) { /* upper */
266 if (c == (UChar )0x97) return FALSE;
267 return TRUE;
268 }
269 else if (c >= (UChar )0xa0 && c <= (UChar )0xbe) { /* lower */
270 if (c == (UChar )'\267') return FALSE;
271 return TRUE;
272 }
273 else if (c == (UChar )0x9f &&
274 (flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) != 0) {
275 return TRUE;
276 }
277 }
278 }
279 }
280
281 return FALSE;
282 }
283 #endif
284
285
286 static int
get_ctype_code_range(OnigCtype ctype,OnigCodePoint * sb_out,const OnigCodePoint * ranges[])287 get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
288 const OnigCodePoint* ranges[])
289 {
290 *sb_out = 0x80;
291 return onigenc_unicode_ctype_code_range(ctype, ranges);
292 }
293
294
295 static UChar*
left_adjust_char_head(const UChar * start,const UChar * s)296 left_adjust_char_head(const UChar* start, const UChar* s)
297 {
298 const UChar *p;
299
300 if (s <= start) return (UChar* )s;
301 p = s;
302
303 while (!utf8_islead(*p) && p > start) p--;
304 return (UChar* )p;
305 }
306
307 static int
get_case_fold_codes_by_str(OnigCaseFoldType flag,const OnigUChar * p,const OnigUChar * end,OnigCaseFoldCodeItem items[])308 get_case_fold_codes_by_str(OnigCaseFoldType flag,
309 const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[])
310 {
311 return onigenc_unicode_get_case_fold_codes_by_str(ONIG_ENCODING_UTF8,
312 flag, p, end, items);
313 }
314
315 OnigEncodingType OnigEncodingUTF8 = {
316 mbc_enc_len,
317 "UTF-8", /* name */
318 6, /* max byte length */
319 1, /* min byte length */
320 is_mbc_newline,
321 mbc_to_code,
322 code_to_mbclen,
323 code_to_mbc,
324 mbc_case_fold,
325 onigenc_unicode_apply_all_case_fold,
326 get_case_fold_codes_by_str,
327 onigenc_unicode_property_name_to_ctype,
328 onigenc_unicode_is_code_ctype,
329 get_ctype_code_range,
330 left_adjust_char_head,
331 onigenc_always_true_is_allowed_reverse_match,
332 NULL, /* init */
333 NULL, /* is_initialized */
334 is_valid_mbc_string
335 };
336
