1 /**********************************************************************
2 unicode.c - Oniguruma (regular expression library)
3 **********************************************************************/
4 /*-
5 * Copyright (c) 2002-2019 K.Kosako
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30 #include "regint.h"
31
32 struct PoolPropertyNameCtype {
33 short int name;
34 short int ctype;
35 };
36
37 #define ONIGENC_IS_UNICODE_ISO_8859_1_CTYPE(code,ctype) \
38 ((EncUNICODE_ISO_8859_1_CtypeTable[code] & CTYPE_TO_BIT(ctype)) != 0)
39
40 static const unsigned short EncUNICODE_ISO_8859_1_CtypeTable[256] = {
41 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
42 0x4008, 0x428c, 0x4289, 0x4288, 0x4288, 0x4288, 0x4008, 0x4008,
43 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
44 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008, 0x4008,
45 0x4284, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
46 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
47 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0, 0x78b0,
48 0x78b0, 0x78b0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x41a0,
49 0x41a0, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x7ca2, 0x74a2,
50 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
51 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2, 0x74a2,
52 0x74a2, 0x74a2, 0x74a2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x51a0,
53 0x41a0, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x78e2, 0x70e2,
54 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
55 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2, 0x70e2,
56 0x70e2, 0x70e2, 0x70e2, 0x41a0, 0x41a0, 0x41a0, 0x41a0, 0x4008,
57 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0288, 0x0008, 0x0008,
58 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008,
59 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008,
60 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008, 0x0008,
61 0x0284, 0x01a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0, 0x00a0,
62 0x00a0, 0x00a0, 0x30e2, 0x01a0, 0x00a0, 0x00a8, 0x00a0, 0x00a0,
63 0x00a0, 0x00a0, 0x10a0, 0x10a0, 0x00a0, 0x30e2, 0x00a0, 0x01a0,
64 0x00a0, 0x10a0, 0x30e2, 0x01a0, 0x10a0, 0x10a0, 0x10a0, 0x01a0,
65 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
66 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2,
67 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x00a0,
68 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x34a2, 0x30e2,
69 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
70 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2,
71 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x00a0,
72 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2
73 };
74
75 #include "st.h"
76
77 #include "unicode_fold_data.c"
78
79 extern int
onigenc_unicode_mbc_case_fold(OnigEncoding enc,OnigCaseFoldType flag ARG_UNUSED,const UChar ** pp,const UChar * end,UChar * fold)80 onigenc_unicode_mbc_case_fold(OnigEncoding enc,
81 OnigCaseFoldType flag ARG_UNUSED, const UChar** pp, const UChar* end,
82 UChar* fold)
83 {
84 const struct ByUnfoldKey* buk;
85
86 OnigCodePoint code;
87 int i, len, rlen;
88 const UChar *p = *pp;
89
90 code = ONIGENC_MBC_TO_CODE(enc, p, end);
91 len = enclen(enc, p);
92 *pp += len;
93
94 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
95 if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
96 if (code == 0x0130) {
97 return ONIGENC_CODE_TO_MBC(enc, 0x0069, fold);
98 }
99 #if 0
100 if (code == 0x0049) {
101 return ONIGENC_CODE_TO_MBC(enc, 0x0131, fold);
102 }
103 #endif
104 }
105 #endif
106
107 buk = onigenc_unicode_unfold_key(code);
108 if (buk != 0) {
109 if (buk->fold_len == 1) {
110 return ONIGENC_CODE_TO_MBC(enc, *FOLDS1_FOLD(buk->index), fold);
111 }
112 else {
113 OnigCodePoint* addr;
114
115 FOLDS_FOLD_ADDR_BUK(buk, addr);
116 rlen = 0;
117 for (i = 0; i < buk->fold_len; i++) {
118 OnigCodePoint c = addr[i];
119 len = ONIGENC_CODE_TO_MBC(enc, c, fold);
120 fold += len;
121 rlen += len;
122 }
123 return rlen;
124 }
125 }
126
127 for (i = 0; i < len; i++) {
128 *fold++ = *p++;
129 }
130 return len;
131 }
132
133 static int
apply_case_fold1(int from,int to,OnigApplyAllCaseFoldFunc f,void * arg)134 apply_case_fold1(int from, int to, OnigApplyAllCaseFoldFunc f, void* arg)
135 {
136 int i, j, k, n, r;
137
138 for (i = from; i < to; ) {
139 OnigCodePoint fold = *FOLDS1_FOLD(i);
140 n = FOLDS1_UNFOLDS_NUM(i);
141 for (j = 0; j < n; j++) {
142 OnigCodePoint unfold = FOLDS1_UNFOLDS(i)[j];
143
144 r = (*f)(fold, &unfold, 1, arg);
145 if (r != 0) return r;
146 r = (*f)(unfold, &fold, 1, arg);
147 if (r != 0) return r;
148
149 for (k = 0; k < j; k++) {
150 OnigCodePoint unfold2 = FOLDS1_UNFOLDS(i)[k];
151 r = (*f)(unfold, &unfold2, 1, arg);
152 if (r != 0) return r;
153 r = (*f)(unfold2, &unfold, 1, arg);
154 if (r != 0) return r;
155 }
156 }
157
158 i = FOLDS1_NEXT_INDEX(i);
159 }
160
161 return 0;
162 }
163
164 static int
apply_case_fold2(int from,int to,OnigApplyAllCaseFoldFunc f,void * arg)165 apply_case_fold2(int from, int to, OnigApplyAllCaseFoldFunc f, void* arg)
166 {
167 int i, j, k, n, r;
168
169 for (i = from; i < to; ) {
170 OnigCodePoint* fold = FOLDS2_FOLD(i);
171 n = FOLDS2_UNFOLDS_NUM(i);
172 for (j = 0; j < n; j++) {
173 OnigCodePoint unfold = FOLDS2_UNFOLDS(i)[j];
174
175 r = (*f)(unfold, fold, 2, arg);
176 if (r != 0) return r;
177
178 for (k = 0; k < j; k++) {
179 OnigCodePoint unfold2 = FOLDS2_UNFOLDS(i)[k];
180 r = (*f)(unfold, &unfold2, 1, arg);
181 if (r != 0) return r;
182 r = (*f)(unfold2, &unfold, 1, arg);
183 if (r != 0) return r;
184 }
185 }
186
187 i = FOLDS2_NEXT_INDEX(i);
188 }
189
190 return 0;
191 }
192
193 static int
apply_case_fold3(int from,int to,OnigApplyAllCaseFoldFunc f,void * arg)194 apply_case_fold3(int from, int to, OnigApplyAllCaseFoldFunc f, void* arg)
195 {
196 int i, j, k, n, r;
197
198 for (i = from; i < to; ) {
199 OnigCodePoint* fold = FOLDS3_FOLD(i);
200 n = FOLDS3_UNFOLDS_NUM(i);
201 for (j = 0; j < n; j++) {
202 OnigCodePoint unfold = FOLDS3_UNFOLDS(i)[j];
203
204 r = (*f)(unfold, fold, 3, arg);
205 if (r != 0) return r;
206
207 for (k = 0; k < j; k++) {
208 OnigCodePoint unfold2 = FOLDS3_UNFOLDS(i)[k];
209 r = (*f)(unfold, &unfold2, 1, arg);
210 if (r != 0) return r;
211 r = (*f)(unfold2, &unfold, 1, arg);
212 if (r != 0) return r;
213 }
214 }
215
216 i = FOLDS3_NEXT_INDEX(i);
217 }
218
219 return 0;
220 }
221
222 extern int
onigenc_unicode_apply_all_case_fold(OnigCaseFoldType flag,OnigApplyAllCaseFoldFunc f,void * arg)223 onigenc_unicode_apply_all_case_fold(OnigCaseFoldType flag,
224 OnigApplyAllCaseFoldFunc f, void* arg)
225 {
226 int r;
227
228 r = apply_case_fold1(0, FOLDS1_NORMAL_END_INDEX, f, arg);
229 if (r != 0) return r;
230
231 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
232 if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
233 code = 0x0131;
234 r = (*f)(0x0049, &code, 1, arg);
235 if (r != 0) return r;
236 code = 0x0049;
237 r = (*f)(0x0131, &code, 1, arg);
238 if (r != 0) return r;
239
240 code = 0x0130;
241 r = (*f)(0x0069, &code, 1, arg);
242 if (r != 0) return r;
243 code = 0x0069;
244 r = (*f)(0x0130, &code, 1, arg);
245 if (r != 0) return r;
246 }
247 else {
248 #endif
249 r = apply_case_fold1(FOLDS1_NORMAL_END_INDEX, FOLDS1_END_INDEX, f, arg);
250 if (r != 0) return r;
251 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
252 }
253 #endif
254
255 if ((flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) == 0)
256 return 0;
257
258 r = apply_case_fold2(0, FOLDS2_NORMAL_END_INDEX, f, arg);
259 if (r != 0) return r;
260
261 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
262 if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) == 0) {
263 #endif
264 r = apply_case_fold2(FOLDS2_NORMAL_END_INDEX, FOLDS2_END_INDEX, f, arg);
265 if (r != 0) return r;
266 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
267 }
268 #endif
269
270 r = apply_case_fold3(0, FOLDS3_NORMAL_END_INDEX, f, arg);
271 if (r != 0) return r;
272
273 return 0;
274 }
275
276 extern int
onigenc_unicode_get_case_fold_codes_by_str(OnigEncoding enc,OnigCaseFoldType flag,const OnigUChar * p,const OnigUChar * end,OnigCaseFoldCodeItem items[])277 onigenc_unicode_get_case_fold_codes_by_str(OnigEncoding enc,
278 OnigCaseFoldType flag, const OnigUChar* p, const OnigUChar* end,
279 OnigCaseFoldCodeItem items[])
280 {
281 int n, m, i, j, k, len;
282 OnigCodePoint code, codes[3];
283 const struct ByUnfoldKey* buk;
284
285 n = 0;
286
287 code = ONIGENC_MBC_TO_CODE(enc, p, end);
288 len = enclen(enc, p);
289
290 #ifdef USE_UNICODE_CASE_FOLD_TURKISH_AZERI
291 if ((flag & ONIGENC_CASE_FOLD_TURKISH_AZERI) != 0) {
292 if (code == 0x0049) {
293 items[0].byte_len = len;
294 items[0].code_len = 1;
295 items[0].code[0] = 0x0131;
296 return 1;
297 }
298 else if (code == 0x0130) {
299 items[0].byte_len = len;
300 items[0].code_len = 1;
301 items[0].code[0] = 0x0069;
302 return 1;
303 }
304 else if (code == 0x0131) {
305 items[0].byte_len = len;
306 items[0].code_len = 1;
307 items[0].code[0] = 0x0049;
308 return 1;
309 }
310 else if (code == 0x0069) {
311 items[0].byte_len = len;
312 items[0].code_len = 1;
313 items[0].code[0] = 0x0130;
314 return 1;
315 }
316 }
317 #endif
318
319 buk = onigenc_unicode_unfold_key(code);
320 if (buk != 0) {
321 if (buk->fold_len == 1) {
322 int un;
323 items[0].byte_len = len;
324 items[0].code_len = 1;
325 items[0].code[0] = *FOLDS1_FOLD(buk->index);
326 n++;
327
328 un = FOLDS1_UNFOLDS_NUM(buk->index);
329 for (i = 0; i < un; i++) {
330 OnigCodePoint unfold = FOLDS1_UNFOLDS(buk->index)[i];
331 if (unfold != code) {
332 items[n].byte_len = len;
333 items[n].code_len = 1;
334 items[n].code[0] = unfold;
335 n++;
336 }
337 }
338 code = items[0].code[0]; /* for multi-code to unfold search. */
339 }
340 else if ((flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) != 0) {
341 OnigCodePoint cs[3][4];
342 int fn, ncs[3];
343
344 if (buk->fold_len == 2) {
345 m = FOLDS2_UNFOLDS_NUM(buk->index);
346 for (i = 0; i < m; i++) {
347 OnigCodePoint unfold = FOLDS2_UNFOLDS(buk->index)[i];
348 if (unfold == code) continue;
349
350 items[n].byte_len = len;
351 items[n].code_len = 1;
352 items[n].code[0] = unfold;
353 n++;
354 }
355
356 for (fn = 0; fn < 2; fn++) {
357 int index;
358 cs[fn][0] = FOLDS2_FOLD(buk->index)[fn];
359 ncs[fn] = 1;
360 index = onigenc_unicode_fold1_key(&cs[fn][0]);
361 if (index >= 0) {
362 int m = FOLDS1_UNFOLDS_NUM(index);
363 for (i = 0; i < m; i++) {
364 cs[fn][i+1] = FOLDS1_UNFOLDS(index)[i];
365 }
366 ncs[fn] += m;
367 }
368 }
369
370 for (i = 0; i < ncs[0]; i++) {
371 for (j = 0; j < ncs[1]; j++) {
372 items[n].byte_len = len;
373 items[n].code_len = 2;
374 items[n].code[0] = cs[0][i];
375 items[n].code[1] = cs[1][j];
376 n++;
377 }
378 }
379 }
380 else { /* fold_len == 3 */
381 m = FOLDS3_UNFOLDS_NUM(buk->index);
382 for (i = 0; i < m; i++) {
383 OnigCodePoint unfold = FOLDS3_UNFOLDS(buk->index)[i];
384 if (unfold == code) continue;
385
386 items[n].byte_len = len;
387 items[n].code_len = 1;
388 items[n].code[0] = unfold;
389 n++;
390 }
391
392 for (fn = 0; fn < 3; fn++) {
393 int index;
394 cs[fn][0] = FOLDS3_FOLD(buk->index)[fn];
395 ncs[fn] = 1;
396 index = onigenc_unicode_fold1_key(&cs[fn][0]);
397 if (index >= 0) {
398 int m = FOLDS1_UNFOLDS_NUM(index);
399 for (i = 0; i < m; i++) {
400 cs[fn][i+1] = FOLDS1_UNFOLDS(index)[i];
401 }
402 ncs[fn] += m;
403 }
404 }
405
406 for (i = 0; i < ncs[0]; i++) {
407 for (j = 0; j < ncs[1]; j++) {
408 for (k = 0; k < ncs[2]; k++) {
409 items[n].byte_len = len;
410 items[n].code_len = 3;
411 items[n].code[0] = cs[0][i];
412 items[n].code[1] = cs[1][j];
413 items[n].code[2] = cs[2][k];
414 n++;
415 }
416 }
417 }
418 }
419
420 /* multi char folded code is not head of another folded multi char */
421 return n;
422 }
423 }
424 else {
425 int index = onigenc_unicode_fold1_key(&code);
426 if (index >= 0) {
427 int m = FOLDS1_UNFOLDS_NUM(index);
428 for (i = 0; i < m; i++) {
429 items[n].byte_len = len;
430 items[n].code_len = 1;
431 items[n].code[0] = FOLDS1_UNFOLDS(index)[i];
432 n++;
433 }
434 }
435 }
436
437 if ((flag & INTERNAL_ONIGENC_CASE_FOLD_MULTI_CHAR) == 0)
438 return n;
439
440 p += len;
441 if (p < end) {
442 int clen;
443 int index;
444
445 codes[0] = code;
446 code = ONIGENC_MBC_TO_CODE(enc, p, end);
447
448 buk = onigenc_unicode_unfold_key(code);
449 if (buk != 0 && buk->fold_len == 1) {
450 codes[1] = *FOLDS1_FOLD(buk->index);
451 }
452 else
453 codes[1] = code;
454
455 clen = enclen(enc, p);
456 len += clen;
457
458 index = onigenc_unicode_fold2_key(codes);
459 if (index >= 0) {
460 m = FOLDS2_UNFOLDS_NUM(index);
461 for (i = 0; i < m; i++) {
462 items[n].byte_len = len;
463 items[n].code_len = 1;
464 items[n].code[0] = FOLDS2_UNFOLDS(index)[i];
465 n++;
466 }
467 }
468
469 p += clen;
470 if (p < end) {
471 code = ONIGENC_MBC_TO_CODE(enc, p, end);
472 buk = onigenc_unicode_unfold_key(code);
473 if (buk != 0 && buk->fold_len == 1) {
474 codes[2] = *FOLDS1_FOLD(buk->index);
475 }
476 else
477 codes[2] = code;
478
479 clen = enclen(enc, p);
480 len += clen;
481
482 index = onigenc_unicode_fold3_key(codes);
483 if (index >= 0) {
484 m = FOLDS3_UNFOLDS_NUM(index);
485 for (i = 0; i < m; i++) {
486 items[n].byte_len = len;
487 items[n].code_len = 1;
488 items[n].code[0] = FOLDS3_UNFOLDS(index)[i];
489 n++;
490 }
491 }
492 }
493 }
494
495 return n;
496 }
497
498 #ifdef USE_UNICODE_PROPERTIES
499 #include "unicode_property_data.c"
500 #else
501 #include "unicode_property_data_posix.c"
502 #endif
503
504
505 #ifdef USE_UNICODE_WORD_BREAK
506
507 enum WB_TYPE {
508 WB_Any = 0,
509 WB_ALetter,
510 WB_CR,
511 WB_Double_Quote,
512 WB_Extend,
513 WB_ExtendNumLet,
514 WB_Format,
515 WB_Hebrew_Letter,
516 WB_Katakana,
517 WB_LF,
518 WB_MidLetter,
519 WB_MidNum,
520 WB_MidNumLet,
521 WB_Newline,
522 WB_Numeric,
523 WB_Regional_Indicator,
524 WB_Single_Quote,
525 WB_WSegSpace,
526 WB_ZWJ,
527 };
528
529 typedef struct {
530 OnigCodePoint start;
531 OnigCodePoint end;
532 enum WB_TYPE type;
533 } WB_RANGE_TYPE;
534
535 #include "unicode_wb_data.c"
536
537 static enum WB_TYPE
wb_get_type(OnigCodePoint code)538 wb_get_type(OnigCodePoint code)
539 {
540 OnigCodePoint low, high, x;
541 enum WB_TYPE type;
542
543 for (low = 0, high = (OnigCodePoint )WB_RANGE_NUM; low < high; ) {
544 x = (low + high) >> 1;
545 if (code > WB_RANGES[x].end)
546 low = x + 1;
547 else
548 high = x;
549 }
550
551 type = (low < (OnigCodePoint )WB_RANGE_NUM &&
552 code >= WB_RANGES[low].start) ?
553 WB_RANGES[low].type : WB_Any;
554
555 return type;
556 }
557
558 #define IS_WB_IGNORE_TAIL(t) ((t) == WB_Extend || (t) == WB_Format || (t) == WB_ZWJ)
559 #define IS_WB_AHLetter(t) ((t) == WB_ALetter || (t) == WB_Hebrew_Letter)
560 #define IS_WB_MidNumLetQ(t) ((t) == WB_MidNumLet || (t) == WB_Single_Quote)
561
562 static int
wb_get_next_main_code(OnigEncoding enc,UChar * p,const UChar * end,OnigCodePoint * rcode,enum WB_TYPE * rtype)563 wb_get_next_main_code(OnigEncoding enc, UChar* p, const UChar* end,
564 OnigCodePoint* rcode, enum WB_TYPE* rtype)
565 {
566 OnigCodePoint code;
567 enum WB_TYPE type;
568
569 while (TRUE) {
570 p += enclen(enc, p);
571 if (p >= end) break;
572
573 code = ONIGENC_MBC_TO_CODE(enc, p, end);
574 type = wb_get_type(code);
575 if (! IS_WB_IGNORE_TAIL(type)) {
576 *rcode = code;
577 *rtype = type;
578 return 1;
579 }
580 }
581
582 return 0;
583 }
584
585 extern int
onigenc_wb_is_break_position(OnigEncoding enc,UChar * p,UChar * prev,const UChar * start,const UChar * end)586 onigenc_wb_is_break_position(OnigEncoding enc, UChar* p, UChar* prev,
587 const UChar* start, const UChar* end)
588 {
589 int r;
590 UChar* pp;
591 OnigCodePoint cfrom;
592 OnigCodePoint cfrom2;
593 OnigCodePoint cto;
594 OnigCodePoint cto2;
595 enum WB_TYPE from;
596 enum WB_TYPE from2;
597 enum WB_TYPE to;
598 enum WB_TYPE to2;
599
600 /* WB1: sot / Any */
601 if (p == start) return TRUE;
602 /* WB2: Any / eot */
603 if (p == end) return TRUE;
604
605 if (IS_NULL(prev)) {
606 prev = onigenc_get_prev_char_head(enc, start, p);
607 if (IS_NULL(prev)) return TRUE;
608 }
609
610 cfrom = ONIGENC_MBC_TO_CODE(enc, prev, end);
611 cto = ONIGENC_MBC_TO_CODE(enc, p, end);
612
613 from = wb_get_type(cfrom);
614 to = wb_get_type(cto);
615
616 /* short cut */
617 if (from == 0 && to == 0) goto WB999;
618
619 /* WB3: CR + LF */
620 if (from == WB_CR && to == WB_LF) return FALSE;
621
622 /* WB3a: (Newline|CR|LF) / */
623 if (from == WB_Newline || from == WB_CR || from == WB_LF) return TRUE;
624 /* WB3b: / (Newline|CR|LF) */
625 if (to == WB_Newline || to == WB_CR || to == WB_LF) return TRUE;
626
627 /* WB3c: ZWJ + {Extended_Pictographic} */
628 if (from == WB_ZWJ) {
629 if (onigenc_unicode_is_code_ctype(cto, PROP_INDEX_EXTENDEDPICTOGRAPHIC))
630 return FALSE;
631 }
632
633 /* WB3d: WSegSpace + WSegSpace */
634 if (from == WB_WSegSpace && to == WB_WSegSpace) return FALSE;
635
636 /* WB4: X (Extend|Format|ZWJ)* -> X */
637 if (IS_WB_IGNORE_TAIL(to)) return FALSE;
638 if (IS_WB_IGNORE_TAIL(from)) {
639 while ((pp = onigenc_get_prev_char_head(enc, start, prev)) != NULL) {
640 prev = pp;
641 cfrom = ONIGENC_MBC_TO_CODE(enc, prev, end);
642 from = wb_get_type(cfrom);
643 if (! IS_WB_IGNORE_TAIL(from))
644 break;
645 }
646 }
647
648 if (IS_WB_AHLetter(from)) {
649 /* WB5: AHLetter + AHLetter */
650 if (IS_WB_AHLetter(to)) return FALSE;
651
652 /* WB6: AHLetter + (MidLetter | MidNumLetQ) AHLetter */
653 if (to == WB_MidLetter || IS_WB_MidNumLetQ(to)) {
654 r = wb_get_next_main_code(enc, p, end, &cto2, &to2);
655 if (r == 1) {
656 if (IS_WB_AHLetter(to2)) return FALSE;
657 }
658 }
659 }
660
661 /* WB7: AHLetter (MidLetter | MidNumLetQ) + AHLetter */
662 if (from == WB_MidLetter || IS_WB_MidNumLetQ(from)) {
663 if (IS_WB_AHLetter(to)) {
664 from2 = WB_Any;
665 while ((pp = onigenc_get_prev_char_head(enc, start, prev)) != NULL) {
666 prev = pp;
667 cfrom2 = ONIGENC_MBC_TO_CODE(enc, prev, end);
668 from2 = wb_get_type(cfrom2);
669 if (! IS_WB_IGNORE_TAIL(from2))
670 break;
671 }
672
673 if (IS_WB_AHLetter(from2)) return FALSE;
674 }
675 }
676
677 if (from == WB_Hebrew_Letter) {
678 /* WB7a: Hebrew_Letter + Single_Quote */
679 if (to == WB_Single_Quote) return FALSE;
680
681 /* WB7b: Hebrew_Letter + Double_Quote Hebrew_Letter */
682 if (to == WB_Double_Quote) {
683 r = wb_get_next_main_code(enc, p, end, &cto2, &to2);
684 if (r == 1) {
685 if (to2 == WB_Hebrew_Letter) return FALSE;
686 }
687 }
688 }
689
690 /* WB7c: Hebrew_Letter Double_Quote + Hebrew_Letter */
691 if (from == WB_Double_Quote) {
692 if (to == WB_Hebrew_Letter) {
693 from2 = WB_Any;
694 while ((pp = onigenc_get_prev_char_head(enc, start, prev)) != NULL) {
695 prev = pp;
696 cfrom2 = ONIGENC_MBC_TO_CODE(enc, prev, end);
697 from2 = wb_get_type(cfrom2);
698 if (! IS_WB_IGNORE_TAIL(from2))
699 break;
700 }
701
702 if (from2 == WB_Hebrew_Letter) return FALSE;
703 }
704 }
705
706 if (to == WB_Numeric) {
707 /* WB8: Numeric + Numeric */
708 if (from == WB_Numeric) return FALSE;
709
710 /* WB9: AHLetter + Numeric */
711 if (IS_WB_AHLetter(from)) return FALSE;
712
713 /* WB11: Numeric (MidNum | MidNumLetQ) + Numeric */
714 if (from == WB_MidNum || IS_WB_MidNumLetQ(from)) {
715 from2 = WB_Any;
716 while ((pp = onigenc_get_prev_char_head(enc, start, prev)) != NULL) {
717 prev = pp;
718 cfrom2 = ONIGENC_MBC_TO_CODE(enc, prev, end);
719 from2 = wb_get_type(cfrom2);
720 if (! IS_WB_IGNORE_TAIL(from2))
721 break;
722 }
723
724 if (from2 == WB_Numeric) return FALSE;
725 }
726 }
727
728 if (from == WB_Numeric) {
729 /* WB10: Numeric + AHLetter */
730 if (IS_WB_AHLetter(to)) return FALSE;
731
732 /* WB12: Numeric + (MidNum | MidNumLetQ) Numeric */
733 if (to == WB_MidNum || IS_WB_MidNumLetQ(to)) {
734 r = wb_get_next_main_code(enc, p, end, &cto2, &to2);
735 if (r == 1) {
736 if (to2 == WB_Numeric) return FALSE;
737 }
738 }
739 }
740
741 /* WB13: Katakana + Katakana */
742 if (from == WB_Katakana && to == WB_Katakana) return FALSE;
743
744 /* WB13a: (AHLetter | Numeric | Katakana | ExtendNumLet) + ExtendNumLet */
745 if (IS_WB_AHLetter(from) || from == WB_Numeric || from == WB_Katakana
746 || from == WB_ExtendNumLet) {
747 if (to == WB_ExtendNumLet) return FALSE;
748 }
749
750 /* WB13b: ExtendNumLet + (AHLetter | Numeric | Katakana) */
751 if (from == WB_ExtendNumLet) {
752 if (IS_WB_AHLetter(to) || to == WB_Numeric || to == WB_Katakana)
753 return FALSE;
754 }
755
756
757 /* WB15: sot (RI RI)* RI + RI */
758 /* WB16: [^RI] (RI RI)* RI + RI */
759 if (from == WB_Regional_Indicator && to == WB_Regional_Indicator) {
760 int n = 0;
761 while ((prev = onigenc_get_prev_char_head(enc, start, prev)) != NULL) {
762 cfrom2 = ONIGENC_MBC_TO_CODE(enc, prev, end);
763 from2 = wb_get_type(cfrom2);
764 if (from2 != WB_Regional_Indicator)
765 break;
766
767 n++;
768 }
769 if ((n % 2) == 0) return FALSE;
770 }
771
772 WB999:
773 /* WB999: Any / Any */
774 return TRUE;
775 }
776
777 #endif /* USE_UNICODE_WORD_BREAK */
778
779
780 #ifdef USE_UNICODE_EXTENDED_GRAPHEME_CLUSTER
781
782 enum EGCB_BREAK_TYPE {
783 EGCB_NOT_BREAK = 0,
784 EGCB_BREAK = 1,
785 EGCB_BREAK_UNDEF_GB11 = 2,
786 EGCB_BREAK_UNDEF_RI_RI = 3
787 };
788
789 enum EGCB_TYPE {
790 EGCB_Other = 0,
791 EGCB_CR = 1,
792 EGCB_LF = 2,
793 EGCB_Control = 3,
794 EGCB_Extend = 4,
795 EGCB_Prepend = 5,
796 EGCB_Regional_Indicator = 6,
797 EGCB_SpacingMark = 7,
798 EGCB_ZWJ = 8,
799 #if 0
800 /* obsoleted */
801 EGCB_E_Base = 9,
802 EGCB_E_Base_GAZ = 10,
803 EGCB_E_Modifier = 11,
804 EGCB_Glue_After_Zwj = 12,
805 #endif
806 EGCB_L = 13,
807 EGCB_LV = 14,
808 EGCB_LVT = 15,
809 EGCB_T = 16,
810 EGCB_V = 17
811 };
812
813 typedef struct {
814 OnigCodePoint start;
815 OnigCodePoint end;
816 enum EGCB_TYPE type;
817 } EGCB_RANGE_TYPE;
818
819 #include "unicode_egcb_data.c"
820
821 static enum EGCB_TYPE
egcb_get_type(OnigCodePoint code)822 egcb_get_type(OnigCodePoint code)
823 {
824 OnigCodePoint low, high, x;
825 enum EGCB_TYPE type;
826
827 for (low = 0, high = (OnigCodePoint )EGCB_RANGE_NUM; low < high; ) {
828 x = (low + high) >> 1;
829 if (code > EGCB_RANGES[x].end)
830 low = x + 1;
831 else
832 high = x;
833 }
834
835 type = (low < (OnigCodePoint )EGCB_RANGE_NUM &&
836 code >= EGCB_RANGES[low].start) ?
837 EGCB_RANGES[low].type : EGCB_Other;
838
839 return type;
840 }
841
842 #define IS_CONTROL_CR_LF(code) ((code) <= EGCB_Control && (code) >= EGCB_CR)
843 #define IS_HANGUL(code) ((code) >= EGCB_L)
844
845 /* GB1 and GB2 are outside of this function. */
846 static enum EGCB_BREAK_TYPE
unicode_egcb_is_break_2code(OnigCodePoint from_code,OnigCodePoint to_code)847 unicode_egcb_is_break_2code(OnigCodePoint from_code, OnigCodePoint to_code)
848 {
849 enum EGCB_TYPE from;
850 enum EGCB_TYPE to;
851
852 from = egcb_get_type(from_code);
853 to = egcb_get_type(to_code);
854
855 /* short cut */
856 if (from == 0 && to == 0) goto GB999;
857
858 /* GB3 */
859 if (from == EGCB_CR && to == EGCB_LF) return EGCB_NOT_BREAK;
860 /* GB4 */
861 if (IS_CONTROL_CR_LF(from)) return EGCB_BREAK;
862 /* GB5 */
863 if (IS_CONTROL_CR_LF(to)) return EGCB_BREAK;
864
865 if (IS_HANGUL(from) && IS_HANGUL(to)) {
866 /* GB6 */
867 if (from == EGCB_L && to != EGCB_T) return EGCB_NOT_BREAK;
868 /* GB7 */
869 if ((from == EGCB_LV || from == EGCB_V)
870 && (to == EGCB_V || to == EGCB_T)) return EGCB_NOT_BREAK;
871
872 /* GB8 */
873 if ((to == EGCB_T) && (from == EGCB_LVT || from == EGCB_T))
874 return EGCB_NOT_BREAK;
875
876 goto GB999;
877 }
878
879 /* GB9 */
880 if (to == EGCB_Extend || to == EGCB_ZWJ) return EGCB_NOT_BREAK;
881
882 /* GB9a */
883 if (to == EGCB_SpacingMark) return EGCB_NOT_BREAK;
884 /* GB9b */
885 if (from == EGCB_Prepend) return EGCB_NOT_BREAK;
886
887 /* GB10 removed */
888
889 /* GB11 */
890 if (from == EGCB_ZWJ) {
891 if (onigenc_unicode_is_code_ctype(to_code, PROP_INDEX_EXTENDEDPICTOGRAPHIC))
892 return EGCB_BREAK_UNDEF_GB11;
893
894 goto GB999;
895 }
896
897 /* GB12, GB13 */
898 if (from == EGCB_Regional_Indicator && to == EGCB_Regional_Indicator) {
899 return EGCB_BREAK_UNDEF_RI_RI;
900 }
901
902 GB999:
903 return EGCB_BREAK;
904 }
905
906 #endif /* USE_UNICODE_EXTENDED_GRAPHEME_CLUSTER */
907
908 extern int
onigenc_egcb_is_break_position(OnigEncoding enc,UChar * p,UChar * prev,const UChar * start,const UChar * end)909 onigenc_egcb_is_break_position(OnigEncoding enc, UChar* p, UChar* prev,
910 const UChar* start, const UChar* end)
911 {
912 OnigCodePoint from;
913 OnigCodePoint to;
914 #ifdef USE_UNICODE_EXTENDED_GRAPHEME_CLUSTER
915 enum EGCB_BREAK_TYPE btype;
916 enum EGCB_TYPE type;
917 #endif
918
919 /* GB1 and GB2 */
920 if (p == start) return 1;
921 if (p == end) return 1;
922
923 if (IS_NULL(prev)) {
924 prev = onigenc_get_prev_char_head(enc, start, p);
925 if (IS_NULL(prev)) return 1;
926 }
927
928 from = ONIGENC_MBC_TO_CODE(enc, prev, end);
929 to = ONIGENC_MBC_TO_CODE(enc, p, end);
930
931 #ifdef USE_UNICODE_EXTENDED_GRAPHEME_CLUSTER
932 if (! ONIGENC_IS_UNICODE_ENCODING(enc)) {
933 return from != 0x000d || to != 0x000a;
934 }
935
936 btype = unicode_egcb_is_break_2code(from, to);
937 switch (btype) {
938 case EGCB_NOT_BREAK:
939 return 0;
940 break;
941 case EGCB_BREAK:
942 return 1;
943 break;
944
945 case EGCB_BREAK_UNDEF_GB11:
946 while ((prev = onigenc_get_prev_char_head(enc, start, prev)) != NULL) {
947 from = ONIGENC_MBC_TO_CODE(enc, prev, end);
948 if (onigenc_unicode_is_code_ctype(from, PROP_INDEX_EXTENDEDPICTOGRAPHIC))
949 return 0;
950
951 type = egcb_get_type(from);
952 if (type != EGCB_Extend)
953 break;
954 }
955 break;
956
957 case EGCB_BREAK_UNDEF_RI_RI:
958 {
959 int n = 0;
960 while ((prev = onigenc_get_prev_char_head(enc, start, prev)) != NULL) {
961 from = ONIGENC_MBC_TO_CODE(enc, prev, end);
962 type = egcb_get_type(from);
963 if (type != EGCB_Regional_Indicator)
964 break;
965
966 n++;
967 }
968 if ((n % 2) == 0) return 0;
969 }
970 break;
971 }
972
973 return 1;
974
975 #else
976 return from != 0x000d || to != 0x000a;
977 #endif /* USE_UNICODE_EXTENDED_GRAPHEME_CLUSTER */
978 }
979
980
981 #define USER_DEFINED_PROPERTY_MAX_NUM 20
982
983 typedef struct {
984 int ctype;
985 OnigCodePoint* ranges;
986 } UserDefinedPropertyValue;
987
988 static int UserDefinedPropertyNum;
989 static UserDefinedPropertyValue
990 UserDefinedPropertyRanges[USER_DEFINED_PROPERTY_MAX_NUM];
991 static st_table* UserDefinedPropertyTable;
992
993 extern int
onig_unicode_define_user_property(const char * name,OnigCodePoint * ranges)994 onig_unicode_define_user_property(const char* name, OnigCodePoint* ranges)
995 {
996 UserDefinedPropertyValue* e;
997 int r;
998 int i;
999 int n;
1000 int len;
1001 int c;
1002 char* s;
1003 UChar* uname;
1004
1005 if (UserDefinedPropertyNum >= USER_DEFINED_PROPERTY_MAX_NUM)
1006 return ONIGERR_TOO_MANY_USER_DEFINED_OBJECTS;
1007
1008 len = (int )strlen(name);
1009 if (len >= PROPERTY_NAME_MAX_SIZE)
1010 return ONIGERR_TOO_LONG_PROPERTY_NAME;
1011
1012 s = (char* )xmalloc(len + 1);
1013 if (s == 0)
1014 return ONIGERR_MEMORY;
1015
1016 uname = (UChar* )name;
1017 n = 0;
1018 for (i = 0; i < len; i++) {
1019 c = uname[i];
1020 if (c < 0x20 || c >= 0x80) {
1021 xfree(s);
1022 return ONIGERR_INVALID_CHAR_PROPERTY_NAME;
1023 }
1024
1025 if (c != ' ' && c != '-' && c != '_') {
1026 s[n] = c;
1027 n++;
1028 }
1029 }
1030 s[n] = '\0';
1031
1032 if (UserDefinedPropertyTable == 0) {
1033 UserDefinedPropertyTable = onig_st_init_strend_table_with_size(10);
1034 if (IS_NULL(UserDefinedPropertyTable)) {
1035 xfree(s);
1036 return ONIGERR_MEMORY;
1037 }
1038 }
1039
1040 e = UserDefinedPropertyRanges + UserDefinedPropertyNum;
1041 e->ctype = CODE_RANGES_NUM + UserDefinedPropertyNum;
1042 e->ranges = ranges;
1043 r = onig_st_insert_strend(UserDefinedPropertyTable,
1044 (const UChar* )s, (const UChar* )s + n,
1045 (hash_data_type )((void* )e));
1046 if (r < 0) return r;
1047
1048 UserDefinedPropertyNum++;
1049 return 0;
1050 }
1051
1052 extern int
onigenc_unicode_is_code_ctype(OnigCodePoint code,unsigned int ctype)1053 onigenc_unicode_is_code_ctype(OnigCodePoint code, unsigned int ctype)
1054 {
1055 if (
1056 #ifdef USE_UNICODE_PROPERTIES
1057 ctype <= ONIGENC_MAX_STD_CTYPE &&
1058 #endif
1059 code < 256) {
1060 return ONIGENC_IS_UNICODE_ISO_8859_1_CTYPE(code, ctype);
1061 }
1062
1063 if (ctype >= CODE_RANGES_NUM) {
1064 int index = ctype - CODE_RANGES_NUM;
1065 if (index < UserDefinedPropertyNum)
1066 return onig_is_in_code_range((UChar* )UserDefinedPropertyRanges[index].ranges, code);
1067 else
1068 return ONIGERR_TYPE_BUG;
1069 }
1070
1071 return onig_is_in_code_range((UChar* )CodeRanges[ctype], code);
1072 }
1073
1074
1075 extern int
onigenc_unicode_ctype_code_range(OnigCtype ctype,const OnigCodePoint * ranges[])1076 onigenc_unicode_ctype_code_range(OnigCtype ctype, const OnigCodePoint* ranges[])
1077 {
1078 if (ctype >= CODE_RANGES_NUM) {
1079 int index = ctype - CODE_RANGES_NUM;
1080 if (index < UserDefinedPropertyNum) {
1081 *ranges = UserDefinedPropertyRanges[index].ranges;
1082 return 0;
1083 }
1084 else
1085 return ONIGERR_TYPE_BUG;
1086 }
1087
1088 *ranges = CodeRanges[ctype];
1089 return 0;
1090 }
1091
1092 extern int
onigenc_utf16_32_get_ctype_code_range(OnigCtype ctype,OnigCodePoint * sb_out,const OnigCodePoint * ranges[])1093 onigenc_utf16_32_get_ctype_code_range(OnigCtype ctype, OnigCodePoint* sb_out,
1094 const OnigCodePoint* ranges[])
1095 {
1096 *sb_out = 0x00;
1097 return onigenc_unicode_ctype_code_range(ctype, ranges);
1098 }
1099
1100 extern int
onigenc_unicode_property_name_to_ctype(OnigEncoding enc,UChar * name,UChar * end)1101 onigenc_unicode_property_name_to_ctype(OnigEncoding enc, UChar* name, UChar* end)
1102 {
1103 int len;
1104 UChar *p;
1105 OnigCodePoint code;
1106 const struct PoolPropertyNameCtype* pc;
1107 char buf[PROPERTY_NAME_MAX_SIZE];
1108
1109 p = name;
1110 len = 0;
1111 while (p < end) {
1112 code = ONIGENC_MBC_TO_CODE(enc, p, end);
1113 if (code >= 0x80)
1114 return ONIGERR_INVALID_CHAR_PROPERTY_NAME;
1115
1116 if (code != ' ' && code != '-' && code != '_') {
1117 buf[len++] = (char )code;
1118 if (len >= PROPERTY_NAME_MAX_SIZE)
1119 return ONIGERR_INVALID_CHAR_PROPERTY_NAME;
1120 }
1121
1122 p += enclen(enc, p);
1123 }
1124
1125 buf[len] = 0;
1126
1127 if (UserDefinedPropertyTable != 0) {
1128 UserDefinedPropertyValue* e;
1129 e = (UserDefinedPropertyValue* )NULL;
1130 onig_st_lookup_strend(UserDefinedPropertyTable,
1131 (const UChar* )buf, (const UChar* )buf + len,
1132 (hash_data_type* )((void* )(&e)));
1133 if (e != 0) {
1134 return e->ctype;
1135 }
1136 }
1137
1138 pc = unicode_lookup_property_name(buf, len);
1139 if (pc != 0) {
1140 /* fprintf(stderr, "LOOKUP: %s: %d\n", buf, pc->ctype); */
1141 #ifndef USE_UNICODE_PROPERTIES
1142 if (pc->ctype > ONIGENC_MAX_STD_CTYPE)
1143 return ONIGERR_INVALID_CHAR_PROPERTY_NAME;
1144 #endif
1145
1146 return (int )pc->ctype;
1147 }
1148
1149 return ONIGERR_INVALID_CHAR_PROPERTY_NAME;
1150 }
1151
