1 /*
2 * "streamable kanji code filter and converter"
3 * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved.
4 *
5 * LICENSE NOTICES
6 *
7 * This file is part of "streamable kanji code filter and converter",
8 * which is distributed under the terms of GNU Lesser General Public
9 * License (version 2) as published by the Free Software Foundation.
10 *
11 * This software is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with "streamable kanji code filter and converter";
18 * if not, write to the Free Software Foundation, Inc., 59 Temple Place,
19 * Suite 330, Boston, MA 02111-1307 USA
20 *
21 * The author of this file:
22 *
23 */
24 /*
25 * the source code included in this files was separated from mbfilter_cp936.c
26 * by rui hirokawa <hirokawa@php.net> on 11 Aug 2011.
27 *
28 */
29
30 #ifdef HAVE_CONFIG_H
31 #include "config.h"
32 #endif
33
34 #include "mbfilter.h"
35 #include "mbfilter_gb18030.h"
36
37 #include "unicode_table_cp936.h"
38 #include "unicode_table_gb18030.h"
39
40 static int mbfl_filt_ident_gb18030(int c, mbfl_identify_filter *filter);
41
42 static const char *mbfl_encoding_gb18030_aliases[] = {"gb-18030", "gb-18030-2000", NULL};
43
44 const mbfl_encoding mbfl_encoding_gb18030 = {
45 mbfl_no_encoding_gb18030,
46 "GB18030",
47 "GB18030",
48 (const char *(*)[])&mbfl_encoding_gb18030_aliases,
49 NULL,
50 MBFL_ENCTYPE_MBCS | MBFL_ENCTYPE_GL_UNSAFE,
51 &vtbl_gb18030_wchar,
52 &vtbl_wchar_gb18030
53 };
54
55 const struct mbfl_identify_vtbl vtbl_identify_gb18030 = {
56 mbfl_no_encoding_gb18030,
57 mbfl_filt_ident_common_ctor,
58 mbfl_filt_ident_common_dtor,
59 mbfl_filt_ident_gb18030
60 };
61
62 const struct mbfl_convert_vtbl vtbl_gb18030_wchar = {
63 mbfl_no_encoding_gb18030,
64 mbfl_no_encoding_wchar,
65 mbfl_filt_conv_common_ctor,
66 mbfl_filt_conv_common_dtor,
67 mbfl_filt_conv_gb18030_wchar,
68 mbfl_filt_conv_common_flush
69 };
70
71 const struct mbfl_convert_vtbl vtbl_wchar_gb18030 = {
72 mbfl_no_encoding_wchar,
73 mbfl_no_encoding_gb18030,
74 mbfl_filt_conv_common_ctor,
75 mbfl_filt_conv_common_dtor,
76 mbfl_filt_conv_wchar_gb18030,
77 mbfl_filt_conv_common_flush
78 };
79
80 #define CK(statement) do { if ((statement) < 0) return (-1); } while (0)
81
82
83 int
mbfl_bisec_srch(int w,const unsigned short * tbl,int n)84 mbfl_bisec_srch(int w, const unsigned short *tbl, int n)
85 {
86 int k, k1 = 0, k2 = n-1;
87
88 while (k1 < k2) {
89 k = (k1+k2) >> 1;
90 if (w <= tbl[2*k+1]) {
91 k2 = k;
92 } else if (w >= tbl[2*k+2]) {
93 k1 = k + 1;
94 } else {
95 return -1;
96 }
97 }
98 return k1;
99 }
100
101 int
mbfl_bisec_srch2(int w,const unsigned short tbl[],int n)102 mbfl_bisec_srch2(int w, const unsigned short tbl[], int n)
103 {
104 int k, k1 = 0, k2 = n;
105
106 if (w == tbl[0]) {
107 return 0;
108 }
109
110 while (k2 - k1 > 1) {
111 k = (k1 + k2) >> 1;
112 if (w < tbl[k]) {
113 k2 = k;
114 } else if (w > tbl[k]) {
115 k1 = k;
116 } else {
117 return k;
118 }
119 }
120 return -1;
121 }
122
123 /*
124 * GB18030 => wchar
125 */
126 int
mbfl_filt_conv_gb18030_wchar(int c,mbfl_convert_filter * filter)127 mbfl_filt_conv_gb18030_wchar(int c, mbfl_convert_filter *filter)
128 {
129 int k;
130 int c1, c2, c3, w = -1;
131
132 switch (filter->status) {
133 case 0:
134 if (c >= 0 && c < 0x80) { /* latin */
135 CK((*filter->output_function)(c, filter->data));
136 } else if (c == 0x80) { /* euro sign */
137 CK((*filter->output_function)(0x20ac, filter->data));
138 } else if (c == 0xff) {
139 CK((*filter->output_function)(0x00ff, filter->data));
140 } else if (c > 0x80 && c < 0xff) { /* dbcs/qbcs lead byte */
141 filter->status = 1;
142 filter->cache = c;
143 } else {
144 w = c & MBFL_WCSGROUP_MASK;
145 w |= MBFL_WCSGROUP_THROUGH;
146 CK((*filter->output_function)(w, filter->data));
147 }
148 break;
149
150 case 1: /* dbcs/qbcs second byte */
151 c1 = filter->cache;
152 filter->status = 0;
153
154 if (c1 >= 0x81 && c1 <= 0x84 && c >= 0x30 && c <= 0x39) { /* 4 byte range: Unicode BMP */
155 filter->status = 2;
156 filter->cache = (c1 << 8) | c;
157 return c;
158 } else if (c1 >= 0x90 && c1 <= 0xe3 && c >= 0x30 && c <= 0x39) {
159 /* 4 byte range: Unicode 16 planes */
160 filter->status = 2;
161 filter->cache = (c1 << 8) | c;
162 return c;
163 } else if (((c1 >= 0xaa && c1 <= 0xaf) || (c1 >= 0xf8 && c1 <= 0xfe)) &&
164 (c >= 0xa1 && c <= 0xfe)) { /* UDA part1,2: U+E000-U+E4C5 */
165 w = 94*(c1 >= 0xf8 ? c1 - 0xf2 : c1 - 0xaa) + (c - 0xa1) + 0xe000;
166 CK((*filter->output_function)(w, filter->data));
167 } else if (c1 >= 0xa1 && c1 <= 0xa7 && c >= 0x40 && c < 0xa1 && c != 0x7f) {
168 /* UDA part3 : U+E4C6-U+E765*/
169 w = 96*(c1 - 0xa1) + c - (c >= 0x80 ? 0x41 : 0x40) + 0xe4c6;
170 CK((*filter->output_function)(w, filter->data));
171 }
172
173 c2 = (c1 << 8) | c;
174
175 if (w <= 0 &&
176 ((c2 >= 0xa2ab && c2 <= 0xa9f0 + (0xe80f-0xe801)) ||
177 (c2 >= 0xd7fa && c2 <= 0xd7fa + (0xe814-0xe810)) ||
178 (c2 >= 0xfe50 && c2 <= 0xfe80 + (0xe864-0xe844)))) {
179 for (k = 0; k < mbfl_gb18030_pua_tbl_max; k++) {
180 if (c2 >= mbfl_gb18030_pua_tbl[k][2] &&
181 c2 <= mbfl_gb18030_pua_tbl[k][2] + mbfl_gb18030_pua_tbl[k][1]
182 - mbfl_gb18030_pua_tbl[k][0]) {
183 w = c2 - mbfl_gb18030_pua_tbl[k][2] + mbfl_gb18030_pua_tbl[k][0];
184 CK((*filter->output_function)(w, filter->data));
185 break;
186 }
187 }
188 }
189
190 if (w <= 0) {
191 if ((c1 >= 0xa1 && c1 <= 0xa9 && c >= 0xa1 && c <= 0xfe) ||
192 (c1 >= 0xb0 && c1 <= 0xf7 && c >= 0xa1 && c <= 0xfe) ||
193 (c1 >= 0x81 && c1 <= 0xa0 && c >= 0x40 && c <= 0xfe && c != 0x7f) ||
194 (c1 >= 0xaa && c1 <= 0xfe && c >= 0x40 && c <= 0xa0 && c != 0x7f) ||
195 (c1 >= 0xa8 && c1 <= 0xa9 && c >= 0x40 && c <= 0xa0 && c != 0x7f)) {
196 w = (c1 - 0x81)*192 + (c - 0x40);
197 if (w >= 0 && w < cp936_ucs_table_size) {
198 w = cp936_ucs_table[w];
199 } else {
200 w = 0;
201 }
202 if (w <= 0) {
203 w = (c1 << 8) | c;
204 w &= MBFL_WCSPLANE_MASK;
205 w |= MBFL_WCSPLANE_GB18030;
206 }
207 CK((*filter->output_function)(w, filter->data));
208 } else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */
209 CK((*filter->output_function)(c, filter->data));
210 } else {
211 w = (c1 << 8) | c;
212 w &= MBFL_WCSGROUP_MASK;
213 w |= MBFL_WCSGROUP_THROUGH;
214 CK((*filter->output_function)(w, filter->data));
215 }
216 }
217 break;
218 case 2: /* qbcs third byte */
219 c1 = (filter->cache >> 8) & 0xff;
220 c2 = filter->cache & 0xff;
221 filter->status = 0;
222 filter->cache = 0;
223 if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) &&
224 c2 >= 0x30 && c2 <= 0x39 && c >= 0x81 && c <= 0xfe) {
225 filter->cache = (c1 << 16) | (c2 << 8) | c;
226 filter->status = 3;
227 } else {
228 w = (c1 << 16) | (c2 << 8) | c;
229 w &= MBFL_WCSGROUP_MASK;
230 w |= MBFL_WCSGROUP_THROUGH;
231 CK((*filter->output_function)(w, filter->data));
232 }
233 break;
234
235 case 3: /* qbcs fourth byte */
236 c1 = (filter->cache >> 16) & 0xff;
237 c2 = (filter->cache >> 8) & 0xff;
238 c3 = filter->cache & 0xff;
239 filter->status = 0;
240 filter->cache = 0;
241 if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) &&
242 c2 >= 0x30 && c2 <= 0x39 && c3 >= 0x81 && c3 <= 0xfe && c >= 0x30 && c <= 0x39) {
243 if (c1 >= 0x90 && c1 <= 0xe3) {
244 w = ((((c1 - 0x90)*10 + (c2 - 0x30))*126 + (c3 - 0x81)))*10 + (c - 0x30) + 0x10000;
245 } else { /* Unicode BMP */
246 w = (((c1 - 0x81)*10 + (c2 - 0x30))*126 + (c3 - 0x81))*10 + (c - 0x30);
247 if (w >= 0 && w <= 39419) {
248 k = mbfl_bisec_srch(w, mbfl_gb2uni_tbl, mbfl_gb_uni_max);
249 if (k<0) {
250 /* error */
251 w = (c1 << 24) | (c2 << 16) | (c3 << 8) | c;
252 w &= MBFL_WCSGROUP_MASK;
253 w |= MBFL_WCSGROUP_THROUGH;
254 CK((*filter->output_function)(w, filter->data));
255 return c;
256 }
257 w += mbfl_gb_uni_ofst[k];
258 } else {
259 w = (c1 << 24) | (c2 << 16) | (c3 << 8) | c;
260 w &= MBFL_WCSGROUP_MASK;
261 w |= MBFL_WCSGROUP_THROUGH;
262 CK((*filter->output_function)(w, filter->data));
263 return c;
264 }
265 }
266 CK((*filter->output_function)(w, filter->data));
267 } else {
268 w = (c1 << 24) | (c2 << 16) | (c3 << 8) | c;
269 w &= MBFL_WCSGROUP_MASK;
270 w |= MBFL_WCSGROUP_THROUGH;
271 CK((*filter->output_function)(w, filter->data));
272 }
273 break;
274
275 default:
276 filter->status = 0;
277 break;
278 }
279
280 return c;
281 }
282
283 /*
284 * wchar => GB18030
285 */
286 int
mbfl_filt_conv_wchar_gb18030(int c,mbfl_convert_filter * filter)287 mbfl_filt_conv_wchar_gb18030(int c, mbfl_convert_filter *filter)
288 {
289 int k, k1, k2;
290 int c1, s = 0, s1 = 0;
291
292 if (c >= ucs_a1_cp936_table_min && c < ucs_a1_cp936_table_max) {
293 s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min];
294 } else if (c >= ucs_a2_cp936_table_min && c < ucs_a2_cp936_table_max) {
295 s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min];
296 } else if (c >= ucs_a3_cp936_table_min && c < ucs_a3_cp936_table_max) {
297 s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min];
298 } else if (c >= ucs_i_cp936_table_min && c < ucs_i_cp936_table_max) {
299 s = ucs_i_cp936_table[c - ucs_i_cp936_table_min];
300 } else if (c >= ucs_ci_cp936_table_min && c < ucs_ci_cp936_table_max) {
301 /* U+F900-FA2F CJK Compatibility Ideographs */
302 if (c == 0xf92c) {
303 s = 0xfd9c;
304 } else if (c == 0xf979) {
305 s = 0xfd9d;
306 } else if (c == 0xf995) {
307 s = 0xfd9e;
308 } else if (c == 0xf9e7) {
309 s = 0xfd9f;
310 } else if (c == 0xf9f1) {
311 s = 0xfda0;
312 } else if (c >= 0xfa0c && c <= 0xfa29) {
313 s = ucs_ci_s_cp936_table[c - 0xfa0c];
314 }
315 } else if (c >= ucs_cf_cp936_table_min && c < ucs_cf_cp936_table_max) {
316 /* FE30h CJK Compatibility Forms */
317 s = ucs_cf_cp936_table[c - ucs_cf_cp936_table_min];
318 } else if (c >= ucs_sfv_cp936_table_min && c < ucs_sfv_cp936_table_max) {
319 /* U+FE50-FE6F Small Form Variants */
320 s = ucs_sfv_cp936_table[c - ucs_sfv_cp936_table_min];
321 } else if (c >= ucs_hff_cp936_table_min && c < ucs_hff_cp936_table_max) {
322 /* U+FF00-FFFF HW/FW Forms */
323 if (c == 0xff04) {
324 s = 0xa1e7;
325 } else if (c == 0xff5e) {
326 s = 0xa1ab;
327 } else if (c >= 0xff01 && c <= 0xff5d) {
328 s = c - 0xff01 + 0xa3a1;
329 } else if (c >= 0xffe0 && c <= 0xffe5) {
330 s = ucs_hff_s_cp936_table[c-0xffe0];
331 }
332 }
333
334 if (c == 0x20ac) { /* euro-sign */
335 s = 0xa2e3;
336 }
337
338 if (s <= 0 && c >= mbfl_gb18030_c_tbl_key[0] &&
339 c <= mbfl_gb18030_c_tbl_key[mbfl_gb18030_c_tbl_max-1]) {
340 k1 = mbfl_bisec_srch2(c, mbfl_gb18030_c_tbl_key, mbfl_gb18030_c_tbl_max);
341 if (k1 >= 0) {
342 s = mbfl_gb18030_c_tbl_val[k1];
343 }
344 }
345
346 if (c >= 0xe000 && c <= 0xe864) { /* PUA */
347 if (c < 0xe766) {
348 if (c < 0xe4c6) {
349 c1 = c - 0xe000;
350 s = (c1 % 94) + 0xa1; c1 /= 94;
351 s |= (c1 < 0x06 ? c1 + 0xaa : c1 + 0xf2) << 8;
352 } else {
353 c1 = c - 0xe4c6;
354 s = ((c1 / 96) + 0xa1) << 8; c1 %= 96;
355 s |= c1 + (c1 >= 0x3f ? 0x41 : 0x40);
356 }
357 } else {
358 /* U+E766..U+E864 */
359 k1 = 0; k2 = mbfl_gb18030_pua_tbl_max;
360 while (k1 < k2) {
361 k = (k1 + k2) >> 1;
362 if (c < mbfl_gb18030_pua_tbl[k][0]) {
363 k2 = k;
364 } else if (c > mbfl_gb18030_pua_tbl[k][1]) {
365 k1 = k + 1;
366 } else {
367 s = c - mbfl_gb18030_pua_tbl[k][0] + mbfl_gb18030_pua_tbl[k][2];
368 break;
369 }
370 }
371 }
372 }
373
374 if (s <= 0 && c >= 0x0080 && c <= 0xffff) { /* BMP */
375 s = mbfl_bisec_srch(c, mbfl_uni2gb_tbl, mbfl_gb_uni_max);
376 if (s >= 0) {
377 c1 = c - mbfl_gb_uni_ofst[s];
378 s = (c1 % 10) + 0x30; c1 /= 10;
379 s |= ((c1 % 126) + 0x81) << 8; c1 /= 126;
380 s |= ((c1 % 10) + 0x30) << 16; c1 /= 10;
381 s1 = c1 + 0x81;
382 }
383 } else if (c >= 0x10000 && c <= 0x10ffff) { /* Code set 3: Unicode U+10000..U+10FFFF */
384 c1 = c - 0x10000;
385 s = (c1 % 10) + 0x30; c1 /= 10;
386 s |= ((c1 % 126) + 0x81) << 8; c1 /= 126;
387 s |= ((c1 % 10) + 0x30) << 16; c1 /= 10;
388 s1 = c1 + 0x90;
389 }
390
391 if (s <= 0) {
392 c1 = c & ~MBFL_WCSPLANE_MASK;
393 if (c1 == MBFL_WCSPLANE_WINCP936) {
394 s = c & MBFL_WCSPLANE_MASK;
395 }
396 if (c == 0) {
397 s = 0;
398 } else if (s <= 0) {
399 s = -1;
400 }
401 }
402 if (s >= 0) {
403 if (s <= 0x80) { /* latin */
404 CK((*filter->output_function)(s, filter->data));
405 } else if (s1 > 0) { /* qbcs */
406 CK((*filter->output_function)(s1 & 0xff, filter->data));
407 CK((*filter->output_function)((s >> 16) & 0xff, filter->data));
408 CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
409 CK((*filter->output_function)(s & 0xff, filter->data));
410 } else { /* dbcs */
411 CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
412 CK((*filter->output_function)(s & 0xff, filter->data));
413 }
414 } else {
415 if (filter->illegal_mode != MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE) {
416 CK(mbfl_filt_conv_illegal_output(c, filter));
417 }
418 }
419
420 return c;
421 }
422
mbfl_filt_ident_gb18030(int c,mbfl_identify_filter * filter)423 static int mbfl_filt_ident_gb18030(int c, mbfl_identify_filter *filter)
424 {
425 int c1;
426
427 c1 = (filter->status >> 8) & 0xff;
428 filter->status &= 0xff;
429
430 if (filter->status == 0) {
431 if (c <= 0x80 || c == 0xff) {
432 filter->status = 0;
433 } else {
434 filter->status = 1;
435 filter->status |= (c << 8);
436 }
437 } else if (filter->status == 1) { /* dbcs/qbcs 2nd byte */
438 if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) && c >= 0x30 && c <= 0x39) { /* qbcs */
439 filter->status = 2;
440 } else if (((c1 >= 0xaa && c1 <= 0xaf) || (c1 >= 0xf8 && c1 <= 0xfe)) && (c >= 0xa1 && c <= 0xfe)) {
441 filter->status = 0; /* UDA part 1,2 */
442 } else if (c1 >= 0xa1 && c1 <= 0xa7 && c >= 0x40 && c < 0xa1 && c != 0x7f) {
443 filter->status = 0; /* UDA part 3 */
444 } else if ((c1 >= 0xa1 && c1 <= 0xa9 && c >= 0xa1 && c <= 0xfe) ||
445 (c1 >= 0xb0 && c1 <= 0xf7 && c >= 0xa1 && c <= 0xfe) ||
446 (c1 >= 0x81 && c1 <= 0xa0 && c >= 0x40 && c <= 0xfe && c != 0x7f) ||
447 (c1 >= 0xaa && c1 <= 0xfe && c >= 0x40 && c <= 0xa0 && c != 0x7f) ||
448 (c1 >= 0xa8 && c1 <= 0xa9 && c >= 0x40 && c <= 0xa0 && c != 0x7f)) {
449 filter->status = 0; /* DBCS */
450 } else {
451 filter->flag = 1; /* bad */
452 filter->status = 0;
453 }
454 } else if (filter->status == 2) { /* qbcs 3rd byte */
455 if (c > 0x80 && c < 0xff) {
456 filter->status = 3;
457 } else {
458 filter->flag = 1; /* bad */
459 filter->status = 0;
460 }
461 } else if (filter->status == 3) { /* qbcs 4th byte */
462 if (c >= 0x30 && c < 0x40) {
463 filter->status = 0;
464 } else {
465 filter->flag = 1; /* bad */
466 filter->status = 0;
467 }
468 } else { /* bad */
469 filter->flag = 1;
470 }
471
472 return c;
473 }
474
