1 /**********************************************************************
2 utf8.c - Oniguruma (regular expression library)
3 **********************************************************************/
4 /*-
5 * Copyright (c) 2002-2007 K.Kosako <sndgk393 AT ybb DOT ne DOT jp>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "regenc.h"
31
32 #define USE_INVALID_CODE_SCHEME
33
34 #ifdef USE_INVALID_CODE_SCHEME
35 /* virtual codepoint values for invalid encoding byte 0xfe and 0xff */
36 #define INVALID_CODE_FE 0xfffffffe
37 #define INVALID_CODE_FF 0xffffffff
38 #define VALID_CODE_LIMIT 0x7fffffff
39 #endif
40
41 #define utf8_islead(c) ((UChar )((c) & 0xc0) != 0x80)
42
43 static const int EncLen_UTF8[] = {
44 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
45 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
46 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
47 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
48 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
49 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
50 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
51 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
52 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
53 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
54 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
55 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
56 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
57 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
58 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
59 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1
60 };
61
62 static int
mbc_enc_len(const UChar * p)63 mbc_enc_len(const UChar* p)
64 {
65 return EncLen_UTF8[*p];
66 }
67
68 static int
is_mbc_newline(const UChar * p,const UChar * end)69 is_mbc_newline(const UChar* p, const UChar* end)
70 {
71 if (p < end) {
72 if (*p == 0x0a) return 1;
73
74 #ifdef USE_UNICODE_ALL_LINE_TERMINATORS
75 #ifndef USE_CRNL_AS_LINE_TERMINATOR
76 if (*p == 0x0d) return 1;
77 #endif
78 if (p + 1 < end) {
79 if (*(p+1) == 0x85 && *p == 0xc2) /* U+0085 */
80 return 1;
81 if (p + 2 < end) {
82 if ((*(p+2) == 0xa8 || *(p+2) == 0xa9)
83 && *(p+1) == 0x80 && *p == 0xe2) /* U+2028, U+2029 */
84 return 1;
85 }
86 }
87 #endif
88 }
89
90 return 0;
91 }
92
93 static OnigCodePoint
mbc_to_code(const UChar * p,const UChar * end)94 mbc_to_code(const UChar* p, const UChar* end)
95 {
96 int c, len;
97 OnigCodePoint n;
98
99 len = mbc_enc_len(p);
100 if (len > end - p) len = end - p;
101
102 c = *p++;
103 if (len > 1) {
104 len--;
105 n = c & ((1 << (6 - len)) - 1);
106 while (len--) {
107 c = *p++;
108 n = (n << 6) | (c & ((1 << 6) - 1));
109 }
110 return n;
111 }
112 else {
113 #ifdef USE_INVALID_CODE_SCHEME
114 if (c > 0xfd) {
115 return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF);
116 }
117 #endif
118 return (OnigCodePoint )c;
119 }
120 }
121
122 static int
code_to_mbclen(OnigCodePoint code)123 code_to_mbclen(OnigCodePoint code)
124 {
125 if ((code & 0xffffff80) == 0) return 1;
126 else if ((code & 0xfffff800) == 0) return 2;
127 else if ((code & 0xffff0000) == 0) return 3;
128 else if ((code & 0xffe00000) == 0) return 4;
129 else if ((code & 0xfc000000) == 0) return 5;
130 else if ((code & 0x80000000) == 0) return 6;
131 #ifdef USE_INVALID_CODE_SCHEME
132 else if (code == INVALID_CODE_FE) return 1;
133 else if (code == INVALID_CODE_FF) return 1;
134 #endif
135 else
136 return ONIGERR_INVALID_CODE_POINT_VALUE;
137 }
138
139 static int
code_to_mbc(OnigCodePoint code,UChar * buf)140 code_to_mbc(OnigCodePoint code, UChar *buf)
141 {
142 #define UTF8_TRAILS(code, shift) (UChar )((((code) >> (shift)) & 0x3f) | 0x80)
143 #define UTF8_TRAIL0(code) (UChar )(((code) & 0x3f) | 0x80)
144
145 if ((code & 0xffffff80) == 0) {
146 *buf = (UChar )code;
147 return 1;
148 }
149 else {
150 UChar *p = buf;
151
152 if ((code & 0xfffff800) == 0) {
153 *p++ = (UChar )(((code>>6)& 0x1f) | 0xc0);
154 }
155 else if ((code & 0xffff0000) == 0) {
156 *p++ = (UChar )(((code>>12) & 0x0f) | 0xe0);
157 *p++ = UTF8_TRAILS(code, 6);
158 }
159 else if ((code & 0xffe00000) == 0) {
160 *p++ = (UChar )(((code>>18) & 0x07) | 0xf0);
161 *p++ = UTF8_TRAILS(code, 12);
162 *p++ = UTF8_TRAILS(code, 6);
163 }
164 else if ((code & 0xfc000000) == 0) {
165 *p++ = (UChar )(((code>>24) & 0x03) | 0xf8);
166 *p++ = UTF8_TRAILS(code, 18);
167 *p++ = UTF8_TRAILS(code, 12);
168 *p++ = UTF8_TRAILS(code, 6);
169 }
170 else if ((code & 0x80000000) == 0) {
171 *p++ = (UChar )(((code>>30) & 0x01) | 0xfc);
172 *p++ = UTF8_TRAILS(code, 24);
173 *p++ = UTF8_TRAILS(code, 18);
174 *p++ = UTF8_TRAILS(code, 12);
175 *p++ = UTF8_TRAILS(code, 6);
176 }
177 #ifdef USE_INVALID_CODE_SCHEME
178 else if (code == INVALID_CODE_FE) {
179 *p = 0xfe;
180 return 1;
181 }
182 else if (code == INVALID_CODE_FF) {
183 *p = 0xff;
184 return 1;
185 }
186 #endif
187 else {
188 return ONIGERR_TOO_BIG_WIDE_CHAR_VALUE;
189 }
190
191 *p++ = UTF8_TRAIL0(code);
192 return p - buf;
193 }
194 }
195
196 static int
mbc_case_fold(OnigCaseFoldType flag,const UChar ** pp,const UChar * end,UChar * fold)197 mbc_case_fold(OnigCaseFoldType flag, const UChar** pp,
198 const UChar* end, UChar* fold)
199 {
200 const UChar* p = *pp;
201
202 if (ONIGENC_IS_MBC_ASCII(p)) {
203 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
204 if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
205 if (*p == 0x49) {
206 *fold++ = 0xc4;
207 *fold = 0xb1;
208 (*pp)++;
209 return 2;
210 }
211 }
212 #endif
213
214 *fold = ONIGENC_ASCII_CODE_TO_LOWER_CASE(*p);
215 (*pp)++;
216 return 1; /* return byte length of converted char to lower */
217 }
218 else {
219 return onigenc_unicode_mbc_case_fold(ONIG_ENCODING_UTF8, flag,
220 pp, end, fold);
221 }
222 }
223
224 #if 0
225 static int
226 is_mbc_ambiguous(OnigCaseFoldType flag, const UChar** pp, const UChar* end)
227 {
228 const UChar* p = *pp;
229
230 if (ONIGENC_IS_MBC_ASCII(p)) {
231 (*pp)++;
232 return ONIGENC_IS_ASCII_CODE_CASE_AMBIG(*p);
233 }
234 else {
235 (*pp) += enclen(ONIG_ENCODING_UTF8, p);
236
237 if (*p == 0xc3) {
238 int c = *(p + 1);
239 if (c >= 0x80) {
240 if (c <= (UChar )0x9e) { /* upper */
241 if (c == (UChar )0x97) return FALSE;
242 return TRUE;
243 }
244 else if (c >= (UChar )0xa0 && c <= (UChar )0xbe) { /* lower */
245 if (c == (UChar )'\267') return FALSE;
246 return TRUE;
247 }
248 else if (c == (UChar )0x9f &&
249 (flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) != 0) {
250 return TRUE;
251 }
252 }
253 }
254 }
255
256 return FALSE;
257 }
258 #endif
259
260
261 static int
get_ctype_code_range(OnigCtype ctype,OnigCodePoint * sb_out,const OnigCodePoint * ranges[])262 get_ctype_code_range(OnigCtype ctype, OnigCodePoint *sb_out,
263 const OnigCodePoint* ranges[])
264 {
265 *sb_out = 0x80;
266 return onigenc_unicode_ctype_code_range(ctype, ranges);
267 }
268
269
270 static UChar*
left_adjust_char_head(const UChar * start,const UChar * s)271 left_adjust_char_head(const UChar* start, const UChar* s)
272 {
273 const UChar *p;
274
275 if (s <= start) return (UChar* )s;
276 p = s;
277
278 while (!utf8_islead(*p) && p > start) p--;
279 return (UChar* )p;
280 }
281
282 static int
get_case_fold_codes_by_str(OnigCaseFoldType flag,const OnigUChar * p,const OnigUChar * end,OnigCaseFoldCodeItem items[])283 get_case_fold_codes_by_str(OnigCaseFoldType flag,
284 const OnigUChar* p, const OnigUChar* end, OnigCaseFoldCodeItem items[])
285 {
286 return onigenc_unicode_get_case_fold_codes_by_str(ONIG_ENCODING_UTF8,
287 flag, p, end, items);
288 }
289
290 OnigEncodingType OnigEncodingUTF8 = {
291 mbc_enc_len,
292 "UTF-8", /* name */
293 6, /* max byte length */
294 1, /* min byte length */
295 is_mbc_newline,
296 mbc_to_code,
297 code_to_mbclen,
298 code_to_mbc,
299 mbc_case_fold,
300 onigenc_unicode_apply_all_case_fold,
301 get_case_fold_codes_by_str,
302 onigenc_unicode_property_name_to_ctype,
303 onigenc_unicode_is_code_ctype,
304 get_ctype_code_range,
305 left_adjust_char_head,
306 onigenc_always_true_is_allowed_reverse_match
307 };
308
