1 /*
2  * "streamable kanji code filter and converter"
3  * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved.
4  *
5  * LICENSE NOTICES
6  *
7  * This file is part of "streamable kanji code filter and converter",
8  * which is distributed under the terms of GNU Lesser General Public
9  * License (version 2) as published by the Free Software Foundation.
10  *
11  * This software is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with "streamable kanji code filter and converter";
18  * if not, write to the Free Software Foundation, Inc., 59 Temple Place,
19  * Suite 330, Boston, MA  02111-1307  USA
20  *
21  * The author of this file:
22  *
23  */
24 /*
25  * the source code included in this files was separated from mbfilter_cp936.c
26  * by rui hirokawa <hirokawa@php.net> on 11 Aug 2011.
27  *
28  */
29 
30 #ifdef HAVE_CONFIG_H
31 #include "config.h"
32 #endif
33 
34 #include "mbfilter.h"
35 #include "mbfilter_gb18030.h"
36 
37 #include "unicode_table_cp936.h"
38 #include "unicode_table_gb18030.h"
39 
40 static int mbfl_filt_ident_gb18030(int c, mbfl_identify_filter *filter);
41 
42 static const char *mbfl_encoding_gb18030_aliases[] = {"gb-18030", "gb-18030-2000", NULL};
43 
44 const mbfl_encoding mbfl_encoding_gb18030 = {
45 	mbfl_no_encoding_gb18030,
46 	"GB18030",
47 	"GB18030",
48 	(const char *(*)[])&mbfl_encoding_gb18030_aliases,
49 	NULL,
50 	MBFL_ENCTYPE_MBCS | MBFL_ENCTYPE_GL_UNSAFE,
51 	&vtbl_gb18030_wchar,
52 	&vtbl_wchar_gb18030
53 };
54 
55 const struct mbfl_identify_vtbl vtbl_identify_gb18030 = {
56 	mbfl_no_encoding_gb18030,
57 	mbfl_filt_ident_common_ctor,
58 	mbfl_filt_ident_common_dtor,
59 	mbfl_filt_ident_gb18030
60 };
61 
62 const struct mbfl_convert_vtbl vtbl_gb18030_wchar = {
63 	mbfl_no_encoding_gb18030,
64 	mbfl_no_encoding_wchar,
65 	mbfl_filt_conv_common_ctor,
66 	mbfl_filt_conv_common_dtor,
67 	mbfl_filt_conv_gb18030_wchar,
68 	mbfl_filt_conv_common_flush
69 };
70 
71 const struct mbfl_convert_vtbl vtbl_wchar_gb18030 = {
72 	mbfl_no_encoding_wchar,
73 	mbfl_no_encoding_gb18030,
74 	mbfl_filt_conv_common_ctor,
75 	mbfl_filt_conv_common_dtor,
76 	mbfl_filt_conv_wchar_gb18030,
77 	mbfl_filt_conv_common_flush
78 };
79 
80 #define CK(statement)	do { if ((statement) < 0) return (-1); } while (0)
81 
82 
83 int
mbfl_bisec_srch(int w,const unsigned short * tbl,int n)84 mbfl_bisec_srch(int w, const unsigned short *tbl, int n)
85 {
86 	int k, k1 = 0, k2 = n-1;
87 
88 	while (k1 < k2) {
89 		k = (k1+k2) >> 1;
90 		if (w <= tbl[2*k+1]) {
91 			k2 = k;
92 		} else if (w >= tbl[2*k+2]) {
93 			k1 = k + 1;
94 		} else {
95 			return -1;
96 		}
97 	}
98 	return k1;
99 }
100 
101 int
mbfl_bisec_srch2(int w,const unsigned short tbl[],int n)102 mbfl_bisec_srch2(int w, const unsigned short tbl[], int n)
103 {
104 	int k, k1 = 0, k2 = n;
105 
106 	if (w == tbl[0]) {
107 		return 0;
108 	}
109 
110 	while (k2 - k1 > 1) {
111 		k = (k1 + k2) >> 1;
112 		if (w < tbl[k]) {
113 			k2 = k;
114 		} else if (w > tbl[k]) {
115 			k1 = k;
116 		} else {
117 			return k;
118 		}
119 	}
120 	return -1;
121 }
122 
123 /*
124  * GB18030 => wchar
125  */
126 int
mbfl_filt_conv_gb18030_wchar(int c,mbfl_convert_filter * filter)127 mbfl_filt_conv_gb18030_wchar(int c, mbfl_convert_filter *filter)
128 {
129 	int k;
130 	int c1, c2, c3, w = -1;
131 
132 	switch (filter->status) {
133 	case 0:
134 		if (c >= 0 && c < 0x80) {	/* latin */
135 			CK((*filter->output_function)(c, filter->data));
136 		} else if (c == 0x80) {	/* euro sign */
137 			CK((*filter->output_function)(0x20ac, filter->data));
138 		} else if (c == 0xff) {
139 			CK((*filter->output_function)(0x00ff, filter->data));
140 		} else if (c > 0x80 && c < 0xff) {	/* dbcs/qbcs lead byte */
141 			filter->status = 1;
142 			filter->cache = c;
143 		} else {
144 			w = c & MBFL_WCSGROUP_MASK;
145 			w |= MBFL_WCSGROUP_THROUGH;
146 			CK((*filter->output_function)(w, filter->data));
147 		}
148 		break;
149 
150 	case 1:		/* dbcs/qbcs second byte */
151 		c1 = filter->cache;
152 		filter->status = 0;
153 
154 		if (c1 >= 0x81 && c1 <= 0x84 && c >= 0x30 && c <= 0x39) { /* 4 byte range: Unicode BMP */
155 			filter->status = 2;
156 			filter->cache = (c1 << 8) | c;
157 			return c;
158 		} else if (c1 >= 0x90 && c1 <= 0xe3 && c >= 0x30 && c <= 0x39) {
159 			/* 4 byte range: Unicode 16 planes */
160 			filter->status = 2;
161 			filter->cache = (c1 << 8) | c;
162 			return c;
163 		} else if (((c1 >= 0xaa && c1 <= 0xaf) || (c1 >= 0xf8 && c1 <= 0xfe)) &&
164 				   (c >= 0xa1 && c <= 0xfe)) { /* UDA part1,2: U+E000-U+E4C5 */
165 			w = 94*(c1 >= 0xf8 ? c1 - 0xf2 : c1 - 0xaa) + (c - 0xa1) + 0xe000;
166 			CK((*filter->output_function)(w, filter->data));
167 		} else if (c1 >= 0xa1 && c1 <= 0xa7 && c >= 0x40 && c < 0xa1 && c != 0x7f) {
168 			/* UDA part3 : U+E4C6-U+E765*/
169 			w = 96*(c1 - 0xa1) + c - (c >= 0x80 ? 0x41 : 0x40) + 0xe4c6;
170 			CK((*filter->output_function)(w, filter->data));
171 		}
172 
173 		c2 = (c1 << 8) | c;
174 
175 		if (w <= 0 &&
176 			((c2 >= 0xa2ab && c2 <= 0xa9f0 + (0xe80f-0xe801)) ||
177 			 (c2 >= 0xd7fa && c2 <= 0xd7fa + (0xe814-0xe810)) ||
178 			 (c2 >= 0xfe50 && c2 <= 0xfe80 + (0xe864-0xe844)))) {
179 			for (k = 0; k < mbfl_gb18030_pua_tbl_max; k++) {
180 				if (c2 >= mbfl_gb18030_pua_tbl[k][2] &&
181 					c2 <= mbfl_gb18030_pua_tbl[k][2] +  mbfl_gb18030_pua_tbl[k][1]
182 					-  mbfl_gb18030_pua_tbl[k][0]) {
183 					w = c2 -  mbfl_gb18030_pua_tbl[k][2] + mbfl_gb18030_pua_tbl[k][0];
184 					CK((*filter->output_function)(w, filter->data));
185 					break;
186 				}
187 			}
188 		}
189 
190 		if (w <= 0) {
191 			if ((c1 >= 0xa1 && c1 <= 0xa9 && c >= 0xa1 && c <= 0xfe) ||
192 				(c1 >= 0xb0 && c1 <= 0xf7 && c >= 0xa1 && c <= 0xfe) ||
193 				(c1 >= 0x81 && c1 <= 0xa0 && c >= 0x40 && c <= 0xfe && c != 0x7f) ||
194 				(c1 >= 0xaa && c1 <= 0xfe && c >= 0x40 && c <= 0xa0 && c != 0x7f) ||
195 				(c1 >= 0xa8 && c1 <= 0xa9 && c >= 0x40 && c <= 0xa0 && c != 0x7f)) {
196 				w = (c1 - 0x81)*192 + (c - 0x40);
197 				if (w >= 0 && w < cp936_ucs_table_size) {
198 					w = cp936_ucs_table[w];
199 				} else {
200 					w = 0;
201 				}
202 				if (w <= 0) {
203 					w = (c1 << 8) | c;
204 					w &= MBFL_WCSPLANE_MASK;
205 					w |= MBFL_WCSPLANE_GB18030;
206 				}
207 				CK((*filter->output_function)(w, filter->data));
208 			} else if ((c >= 0 && c < 0x21) || c == 0x7f) {		/* CTLs */
209 				CK((*filter->output_function)(c, filter->data));
210 			} else {
211 				w = (c1 << 8) | c;
212 				w &= MBFL_WCSGROUP_MASK;
213 				w |= MBFL_WCSGROUP_THROUGH;
214 				CK((*filter->output_function)(w, filter->data));
215 			}
216 		}
217 		break;
218 	case 2: /* qbcs third byte */
219 		c1 = (filter->cache >> 8) & 0xff;
220 		c2 = filter->cache & 0xff;
221 		filter->status = 0;
222 		filter->cache = 0;
223 		if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) &&
224 			c2 >= 0x30 && c2 <= 0x39 && c >= 0x81 && c <= 0xfe) {
225 			filter->cache = (c1 << 16) | (c2 << 8) | c;
226 			filter->status = 3;
227 		} else {
228 			w = (c1 << 16) | (c2 << 8) | c;
229 			w &= MBFL_WCSGROUP_MASK;
230 			w |= MBFL_WCSGROUP_THROUGH;
231 			CK((*filter->output_function)(w, filter->data));
232 		}
233  		break;
234 
235 	case 3: /* qbcs fourth byte */
236 		c1 = (filter->cache >> 16) & 0xff;
237 		c2 = (filter->cache >> 8) & 0xff;
238 		c3 = filter->cache & 0xff;
239 		filter->status = 0;
240 		filter->cache = 0;
241 		if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) &&
242 			c2 >= 0x30 && c2 <= 0x39 && c3 >= 0x81 && c3 <= 0xfe && c >= 0x30 && c <= 0x39) {
243 			if (c1 >= 0x90 && c1 <= 0xe3) {
244 				w = ((((c1 - 0x90)*10 + (c2 - 0x30))*126 + (c3 - 0x81)))*10 + (c - 0x30) + 0x10000;
245 			} else { /* Unicode BMP */
246 				w = (((c1 - 0x81)*10 + (c2 - 0x30))*126 + (c3 - 0x81))*10 + (c - 0x30);
247 				if (w >= 0 && w <= 39419) {
248 					k = mbfl_bisec_srch(w, mbfl_gb2uni_tbl, mbfl_gb_uni_max);
249 					if (k<0) {
250 						/* error */
251 						w = (c1 << 24) | (c2 << 16) | (c3 << 8) | c;
252 						w &= MBFL_WCSGROUP_MASK;
253 						w |= MBFL_WCSGROUP_THROUGH;
254 						CK((*filter->output_function)(w, filter->data));
255 						return c;
256 					}
257 					w += mbfl_gb_uni_ofst[k];
258 				} else {
259 					w = (c1 << 24) | (c2 << 16) | (c3 << 8) | c;
260 					w &= MBFL_WCSGROUP_MASK;
261 					w |= MBFL_WCSGROUP_THROUGH;
262 					CK((*filter->output_function)(w, filter->data));
263 					return c;
264 				}
265 			}
266 			CK((*filter->output_function)(w, filter->data));
267 		} else {
268 			w = (c1 << 24) | (c2 << 16) | (c3 << 8) | c;
269 			w &= MBFL_WCSGROUP_MASK;
270 			w |= MBFL_WCSGROUP_THROUGH;
271 			CK((*filter->output_function)(w, filter->data));
272 		}
273  		break;
274 
275 	default:
276 		filter->status = 0;
277 		break;
278 	}
279 
280 	return c;
281 }
282 
283 /*
284  * wchar => GB18030
285  */
286 int
mbfl_filt_conv_wchar_gb18030(int c,mbfl_convert_filter * filter)287 mbfl_filt_conv_wchar_gb18030(int c, mbfl_convert_filter *filter)
288 {
289 	int k, k1, k2;
290 	int c1, s = 0, s1 = 0;
291 
292 	if (c >= ucs_a1_cp936_table_min && c < ucs_a1_cp936_table_max) {
293 		s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min];
294 	} else if (c >= ucs_a2_cp936_table_min && c < ucs_a2_cp936_table_max) {
295 		s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min];
296 	} else if (c >= ucs_a3_cp936_table_min && c < ucs_a3_cp936_table_max) {
297 		s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min];
298 	} else if (c >= ucs_i_cp936_table_min && c < ucs_i_cp936_table_max) {
299 		s = ucs_i_cp936_table[c - ucs_i_cp936_table_min];
300 	} else if (c >= ucs_ci_cp936_table_min && c < ucs_ci_cp936_table_max) {
301 		/* U+F900-FA2F CJK Compatibility Ideographs */
302 		if (c == 0xf92c) {
303 			s = 0xfd9c;
304 		} else if (c == 0xf979) {
305 			s = 0xfd9d;
306 		} else if (c == 0xf995) {
307 			s = 0xfd9e;
308 		} else if (c == 0xf9e7) {
309 			s = 0xfd9f;
310 		} else if (c == 0xf9f1) {
311 			s = 0xfda0;
312 		} else if (c >= 0xfa0c && c <= 0xfa29) {
313 			s = ucs_ci_s_cp936_table[c - 0xfa0c];
314 		}
315 	} else if (c >= ucs_cf_cp936_table_min && c < ucs_cf_cp936_table_max) {
316 		/* FE30h CJK Compatibility Forms  */
317 		s = ucs_cf_cp936_table[c - ucs_cf_cp936_table_min];
318 	} else if (c >= ucs_sfv_cp936_table_min && c < ucs_sfv_cp936_table_max) {
319 		/* U+FE50-FE6F Small Form Variants */
320 		s = ucs_sfv_cp936_table[c - ucs_sfv_cp936_table_min];
321 	} else if (c >= ucs_hff_cp936_table_min && c < ucs_hff_cp936_table_max) {
322 		/* U+FF00-FFFF HW/FW Forms */
323 		if (c == 0xff04) {
324 			s = 0xa1e7;
325 		} else if (c == 0xff5e) {
326 			s = 0xa1ab;
327 		} else if (c >= 0xff01 && c <= 0xff5d) {
328 			s = c - 0xff01 + 0xa3a1;
329 		} else if (c >= 0xffe0 && c <= 0xffe5) {
330 			s = ucs_hff_s_cp936_table[c-0xffe0];
331 		}
332 	}
333 
334 	if (c == 0x20ac) { /* euro-sign */
335 		s = 0xa2e3;
336 	}
337 
338 	if (s <= 0 && c >= mbfl_gb18030_c_tbl_key[0] &&
339 		c <= mbfl_gb18030_c_tbl_key[mbfl_gb18030_c_tbl_max-1]) {
340 		k1 = mbfl_bisec_srch2(c, mbfl_gb18030_c_tbl_key, mbfl_gb18030_c_tbl_max);
341 		if (k1 >= 0) {
342 			s = mbfl_gb18030_c_tbl_val[k1];
343 		}
344 	}
345 
346 	if (c >= 0xe000 && c <= 0xe864) { /* PUA */
347 		if (c < 0xe766) {
348 			if (c < 0xe4c6) {
349 				c1 = c - 0xe000;
350 				s = (c1 % 94) + 0xa1; c1 /= 94;
351 				s |= (c1 < 0x06 ? c1 + 0xaa : c1 + 0xf2) << 8;
352 			} else {
353 				c1 = c - 0xe4c6;
354 				s = ((c1 / 96) + 0xa1) << 8; c1 %= 96;
355 				s |= c1 + (c1 >= 0x3f ? 0x41 : 0x40);
356 			}
357 		} else {
358 			/* U+E766..U+E864 */
359 			k1 = 0; k2 = mbfl_gb18030_pua_tbl_max;
360 			while (k1 < k2) {
361 				k = (k1 + k2) >> 1;
362 				if (c < mbfl_gb18030_pua_tbl[k][0]) {
363 					k2 = k;
364 				} else if (c > mbfl_gb18030_pua_tbl[k][1]) {
365 					k1 = k + 1;
366 				} else {
367 					s = c - mbfl_gb18030_pua_tbl[k][0] + mbfl_gb18030_pua_tbl[k][2];
368 					break;
369 				}
370 			}
371 		}
372 	}
373 
374 	if (s <= 0 && c >= 0x0080 && c <= 0xffff) { /* BMP */
375 		s = mbfl_bisec_srch(c, mbfl_uni2gb_tbl, mbfl_gb_uni_max);
376 		if (s >= 0) {
377 			c1 = c - mbfl_gb_uni_ofst[s];
378 			s = (c1 % 10) + 0x30; c1 /= 10;
379 			s |= ((c1 % 126) + 0x81) << 8; c1 /= 126;
380 			s |= ((c1 % 10) + 0x30) << 16; c1 /= 10;
381 			s1 = c1 + 0x81;
382 		}
383 	} else if (c >= 0x10000 && c <= 0x10ffff) { /* Code set 3: Unicode U+10000..U+10FFFF */
384 		c1 = c - 0x10000;
385 		s = (c1 % 10) + 0x30; c1 /= 10;
386 		s |= ((c1 % 126) + 0x81) << 8; c1 /= 126;
387 		s |= ((c1 % 10) + 0x30) << 16; c1 /= 10;
388 		s1 = c1 + 0x90;
389 	}
390 
391 	if (s <= 0) {
392 		c1 = c & ~MBFL_WCSPLANE_MASK;
393 		if (c1 == MBFL_WCSPLANE_WINCP936) {
394 			s = c & MBFL_WCSPLANE_MASK;
395 		}
396 		if (c == 0) {
397 			s = 0;
398 		} else if (s <= 0) {
399 			s = -1;
400 		}
401 	}
402 	if (s >= 0) {
403 		if (s <= 0x80) {	/* latin */
404 			CK((*filter->output_function)(s, filter->data));
405 		} else if (s1 > 0) { /* qbcs */
406 			CK((*filter->output_function)(s1 & 0xff, filter->data));
407 			CK((*filter->output_function)((s >> 16) & 0xff, filter->data));
408 			CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
409 			CK((*filter->output_function)(s & 0xff, filter->data));
410 		} else { /* dbcs */
411 			CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
412 			CK((*filter->output_function)(s & 0xff, filter->data));
413 		}
414 	} else {
415 		CK(mbfl_filt_conv_illegal_output(c, filter));
416 	}
417 
418 	return c;
419 }
420 
mbfl_filt_ident_gb18030(int c,mbfl_identify_filter * filter)421 static int mbfl_filt_ident_gb18030(int c, mbfl_identify_filter *filter)
422 {
423 	int c1;
424 
425 	c1 = (filter->status >> 8) & 0xff;
426 	filter->status &= 0xff;
427 
428 	if (filter->status == 0) {
429 		if (c <= 0x80 || c == 0xff) {
430 			filter->status = 0;
431 		} else {
432 			filter->status = 1;
433 			filter->status |= (c << 8);
434 		}
435 	} else if (filter->status == 1) { /* dbcs/qbcs 2nd byte */
436 		if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) && c >= 0x30 && c <= 0x39) { /* qbcs */
437 			filter->status = 2;
438 		} else if (((c1 >= 0xaa && c1 <= 0xaf) || (c1 >= 0xf8 && c1 <= 0xfe)) && (c >= 0xa1 && c <= 0xfe)) {
439 			filter->status = 0; /* UDA part 1,2 */
440 		} else if (c1 >= 0xa1 && c1 <= 0xa7 && c >= 0x40 && c < 0xa1 && c != 0x7f) {
441 			filter->status = 0; /* UDA part 3 */
442 		} else if ((c1 >= 0xa1 && c1 <= 0xa9 && c >= 0xa1 && c <= 0xfe) ||
443 				   (c1 >= 0xb0 && c1 <= 0xf7 && c >= 0xa1 && c <= 0xfe) ||
444 				   (c1 >= 0x81 && c1 <= 0xa0 && c >= 0x40 && c <= 0xfe && c != 0x7f) ||
445 				   (c1 >= 0xaa && c1 <= 0xfe && c >= 0x40 && c <= 0xa0 && c != 0x7f) ||
446 				   (c1 >= 0xa8 && c1 <= 0xa9 && c >= 0x40 && c <= 0xa0 && c != 0x7f)) {
447 			filter->status = 0; /* DBCS */
448 		} else {
449 			filter->flag = 1; /* bad */
450 			filter->status = 0;
451 		}
452 	} else if (filter->status == 2) { /* qbcs 3rd byte */
453 		if (c > 0x80 && c < 0xff) {
454 			filter->status = 3;
455 		} else {
456 			filter->flag = 1; /* bad */
457 			filter->status = 0;
458 		}
459 	} else if (filter->status == 3) { /* qbcs 4th byte */
460 		if (c >= 0x30 && c < 0x40) {
461 			filter->status = 0;
462 		} else {
463 			filter->flag = 1; /* bad */
464 			filter->status = 0;
465 		}
466 	} else {							/* bad */
467 		filter->flag = 1;
468 	}
469 
470 	return c;
471 }
472