1 /*
2 * "streamable kanji code filter and converter"
3 * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved.
4 *
5 * LICENSE NOTICES
6 *
7 * This file is part of "streamable kanji code filter and converter",
8 * which is distributed under the terms of GNU Lesser General Public
9 * License (version 2) as published by the Free Software Foundation.
10 *
11 * This software is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with "streamable kanji code filter and converter";
18 * if not, write to the Free Software Foundation, Inc., 59 Temple Place,
19 * Suite 330, Boston, MA 02111-1307 USA
20 *
21 * The author of this file:
22 *
23 */
24 /*
25 * the source code included in this files was separated from mbfilter_cp936.c
26 * by rui hirokawa <hirokawa@php.net> on 11 Aug 2011.
27 *
28 */
29
30 #include "mbfilter.h"
31 #include "mbfilter_gb18030.h"
32
33 #include "unicode_table_cp936.h"
34 #include "unicode_table_gb18030.h"
35
36 static int mbfl_filt_conv_gb18030_wchar_flush(mbfl_convert_filter *filter);
37
38 static const char *mbfl_encoding_gb18030_aliases[] = {"gb-18030", "gb-18030-2000", NULL};
39
40 const mbfl_encoding mbfl_encoding_gb18030 = {
41 mbfl_no_encoding_gb18030,
42 "GB18030",
43 "GB18030",
44 mbfl_encoding_gb18030_aliases,
45 NULL,
46 MBFL_ENCTYPE_GL_UNSAFE,
47 &vtbl_gb18030_wchar,
48 &vtbl_wchar_gb18030,
49 NULL
50 };
51
52 const struct mbfl_convert_vtbl vtbl_gb18030_wchar = {
53 mbfl_no_encoding_gb18030,
54 mbfl_no_encoding_wchar,
55 mbfl_filt_conv_common_ctor,
56 NULL,
57 mbfl_filt_conv_gb18030_wchar,
58 mbfl_filt_conv_gb18030_wchar_flush,
59 NULL,
60 };
61
62 const struct mbfl_convert_vtbl vtbl_wchar_gb18030 = {
63 mbfl_no_encoding_wchar,
64 mbfl_no_encoding_gb18030,
65 mbfl_filt_conv_common_ctor,
66 NULL,
67 mbfl_filt_conv_wchar_gb18030,
68 mbfl_filt_conv_common_flush,
69 NULL,
70 };
71
72 #define CK(statement) do { if ((statement) < 0) return (-1); } while (0)
73
74 /* `tbl` contains inclusive ranges, each represented by a pair of unsigned shorts */
mbfl_bisec_srch(int w,const unsigned short * tbl,int n)75 int mbfl_bisec_srch(int w, const unsigned short *tbl, int n)
76 {
77 int l = 0, r = n-1;
78 while (l <= r) {
79 int probe = (l + r) >> 1;
80 unsigned short lo = tbl[2 * probe], hi = tbl[(2 * probe) + 1];
81 if (w < lo) {
82 r = probe - 1;
83 } else if (w > hi) {
84 l = probe + 1;
85 } else {
86 return probe;
87 }
88 }
89 return -1;
90 }
91
92 /* `tbl` contains single values, not ranges */
mbfl_bisec_srch2(int w,const unsigned short tbl[],int n)93 int mbfl_bisec_srch2(int w, const unsigned short tbl[], int n)
94 {
95 int l = 0, r = n-1;
96 while (l <= r) {
97 int probe = (l + r) >> 1;
98 unsigned short val = tbl[probe];
99 if (w < val) {
100 r = probe - 1;
101 } else if (w > val) {
102 l = probe + 1;
103 } else {
104 return probe;
105 }
106 }
107 return -1;
108 }
109
mbfl_filt_conv_gb18030_wchar(int c,mbfl_convert_filter * filter)110 int mbfl_filt_conv_gb18030_wchar(int c, mbfl_convert_filter *filter)
111 {
112 int k;
113 int c1, c2, c3, w = -1;
114
115 switch (filter->status) {
116 case 0:
117 if (c >= 0 && c < 0x80) { /* latin */
118 CK((*filter->output_function)(c, filter->data));
119 } else if (c > 0x80 && c < 0xff) { /* dbcs/qbcs lead byte */
120 filter->status = 1;
121 filter->cache = c;
122 } else {
123 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
124 }
125 break;
126
127 case 1: /* dbcs/qbcs second byte */
128 c1 = filter->cache;
129 filter->status = 0;
130
131 if (c1 >= 0x81 && c1 <= 0x84 && c >= 0x30 && c <= 0x39) {
132 /* 4 byte range: Unicode BMP */
133 filter->status = 2;
134 filter->cache = (c1 << 8) | c;
135 return 0;
136 } else if (c1 >= 0x90 && c1 <= 0xe3 && c >= 0x30 && c <= 0x39) {
137 /* 4 byte range: Unicode 16 planes */
138 filter->status = 2;
139 filter->cache = (c1 << 8) | c;
140 return 0;
141 } else if (((c1 >= 0xaa && c1 <= 0xaf) || (c1 >= 0xf8 && c1 <= 0xfe)) && (c >= 0xa1 && c <= 0xfe)) {
142 /* UDA part 1,2: U+E000-U+E4C5 */
143 w = 94*(c1 >= 0xf8 ? c1 - 0xf2 : c1 - 0xaa) + (c - 0xa1) + 0xe000;
144 CK((*filter->output_function)(w, filter->data));
145 } else if (c1 >= 0xa1 && c1 <= 0xa7 && c >= 0x40 && c < 0xa1 && c != 0x7f) {
146 /* UDA part3 : U+E4C6-U+E765*/
147 w = 96*(c1 - 0xa1) + c - (c >= 0x80 ? 0x41 : 0x40) + 0xe4c6;
148 CK((*filter->output_function)(w, filter->data));
149 }
150
151 c2 = (c1 << 8) | c;
152
153 if (w <= 0 &&
154 ((c2 >= 0xa2ab && c2 <= 0xa9f0 + (0xe80f-0xe801)) ||
155 (c2 >= 0xd7fa && c2 <= 0xd7fa + (0xe814-0xe810)) ||
156 (c2 >= 0xfe50 && c2 <= 0xfe80 + (0xe864-0xe844)))) {
157 for (k = 0; k < mbfl_gb18030_pua_tbl_max; k++) {
158 if (c2 >= mbfl_gb18030_pua_tbl[k][2] && c2 <= mbfl_gb18030_pua_tbl[k][2] + mbfl_gb18030_pua_tbl[k][1] - mbfl_gb18030_pua_tbl[k][0]) {
159 w = c2 - mbfl_gb18030_pua_tbl[k][2] + mbfl_gb18030_pua_tbl[k][0];
160 CK((*filter->output_function)(w, filter->data));
161 break;
162 }
163 }
164 }
165
166 if (w <= 0) {
167 if ((c1 >= 0xa1 && c1 <= 0xa9 && c >= 0xa1 && c <= 0xfe) ||
168 (c1 >= 0xb0 && c1 <= 0xf7 && c >= 0xa1 && c <= 0xfe) ||
169 (c1 >= 0x81 && c1 <= 0xa0 && c >= 0x40 && c <= 0xfe && c != 0x7f) ||
170 (c1 >= 0xaa && c1 <= 0xfe && c >= 0x40 && c <= 0xa0 && c != 0x7f) ||
171 (c1 >= 0xa8 && c1 <= 0xa9 && c >= 0x40 && c <= 0xa0 && c != 0x7f)) {
172 w = (c1 - 0x81)*192 + (c - 0x40);
173 if (w >= 0 && w < cp936_ucs_table_size) {
174 w = cp936_ucs_table[w];
175 if (!w)
176 w = MBFL_BAD_INPUT;
177 } else {
178 w = MBFL_BAD_INPUT;
179 }
180 CK((*filter->output_function)(w, filter->data));
181 } else {
182 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
183 }
184 }
185 break;
186
187 case 2: /* qbcs third byte */
188 c1 = (filter->cache >> 8) & 0xff;
189 c2 = filter->cache & 0xff;
190 filter->status = filter->cache = 0;
191 if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) && c2 >= 0x30 && c2 <= 0x39 && c >= 0x81 && c <= 0xfe) {
192 filter->cache = (c1 << 16) | (c2 << 8) | c;
193 filter->status = 3;
194 } else {
195 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
196 }
197 break;
198
199 case 3: /* qbcs fourth byte */
200 c1 = (filter->cache >> 16) & 0xff;
201 c2 = (filter->cache >> 8) & 0xff;
202 c3 = filter->cache & 0xff;
203 filter->status = filter->cache = 0;
204 if (((c1 >= 0x81 && c1 <= 0x84) || (c1 >= 0x90 && c1 <= 0xe3)) && c2 >= 0x30 && c2 <= 0x39 && c3 >= 0x81 && c3 <= 0xfe && c >= 0x30 && c <= 0x39) {
205 if (c1 >= 0x90 && c1 <= 0xe3) {
206 w = ((((c1 - 0x90)*10 + (c2 - 0x30))*126 + (c3 - 0x81)))*10 + (c - 0x30) + 0x10000;
207 if (w > 0x10FFFF) {
208 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
209 return 0;
210 }
211 } else { /* Unicode BMP */
212 w = (((c1 - 0x81)*10 + (c2 - 0x30))*126 + (c3 - 0x81))*10 + (c - 0x30);
213 if (w >= 0 && w <= 39419) {
214 k = mbfl_bisec_srch(w, mbfl_gb2uni_tbl, mbfl_gb_uni_max);
215 if (k < 0) {
216 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
217 return 0;
218 }
219 w += mbfl_gb_uni_ofst[k];
220 } else {
221 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
222 return 0;
223 }
224 }
225 CK((*filter->output_function)(w, filter->data));
226 } else {
227 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
228 }
229 break;
230
231 default:
232 filter->status = 0;
233 break;
234 }
235
236 return 0;
237 }
238
mbfl_filt_conv_gb18030_wchar_flush(mbfl_convert_filter * filter)239 static int mbfl_filt_conv_gb18030_wchar_flush(mbfl_convert_filter *filter)
240 {
241 if (filter->status) {
242 /* multi-byte character was truncated */
243 filter->status = 0;
244 CK((*filter->output_function)(MBFL_BAD_INPUT, filter->data));
245 }
246
247 if (filter->flush_function) {
248 (*filter->flush_function)(filter->data);
249 }
250
251 return 0;
252 }
253
mbfl_filt_conv_wchar_gb18030(int c,mbfl_convert_filter * filter)254 int mbfl_filt_conv_wchar_gb18030(int c, mbfl_convert_filter *filter)
255 {
256 int k, k1, k2;
257 int c1, s = 0, s1 = 0;
258
259 if (c >= ucs_a1_cp936_table_min && c < ucs_a1_cp936_table_max) {
260 if (c == 0x01f9) {
261 s = 0xa8bf;
262 } else {
263 s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min];
264 }
265 } else if (c >= ucs_a2_cp936_table_min && c < ucs_a2_cp936_table_max) {
266 if (c == 0x20ac) { /* euro-sign */
267 s = 0xa2e3;
268 } else {
269 s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min];
270 }
271 } else if (c >= ucs_a3_cp936_table_min && c < ucs_a3_cp936_table_max) {
272 s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min];
273 } else if (c >= ucs_i_cp936_table_min && c < ucs_i_cp936_table_max) {
274 s = ucs_i_cp936_table[c - ucs_i_cp936_table_min];
275 } else if (c >= ucs_ci_cp936_table_min && c < ucs_ci_cp936_table_max) {
276 /* U+F900-FA2F CJK Compatibility Ideographs */
277 if (c == 0xf92c) {
278 s = 0xfd9c;
279 } else if (c == 0xf979) {
280 s = 0xfd9d;
281 } else if (c == 0xf995) {
282 s = 0xfd9e;
283 } else if (c == 0xf9e7) {
284 s = 0xfd9f;
285 } else if (c == 0xf9f1) {
286 s = 0xfda0;
287 } else if (c >= 0xfa0c && c <= 0xfa29) {
288 s = ucs_ci_s_cp936_table[c - 0xfa0c];
289 }
290 } else if (c >= ucs_cf_cp936_table_min && c < ucs_cf_cp936_table_max) {
291 /* FE30h CJK Compatibility Forms */
292 s = ucs_cf_cp936_table[c - ucs_cf_cp936_table_min];
293 } else if (c >= ucs_sfv_cp936_table_min && c < ucs_sfv_cp936_table_max) {
294 /* U+FE50-FE6F Small Form Variants */
295 s = ucs_sfv_cp936_table[c - ucs_sfv_cp936_table_min];
296 } else if (c >= ucs_hff_cp936_table_min && c < ucs_hff_cp936_table_max) {
297 /* U+FF00-FFFF HW/FW Forms */
298 if (c == 0xff04) {
299 s = 0xa1e7;
300 } else if (c == 0xff5e) {
301 s = 0xa1ab;
302 } else if (c >= 0xff01 && c <= 0xff5d) {
303 s = c - 0xff01 + 0xa3a1;
304 } else if (c >= 0xffe0 && c <= 0xffe5) {
305 s = ucs_hff_s_cp936_table[c-0xffe0];
306 }
307 }
308
309 /* While GB18030 and CP936 are very similar, some mappings are different between these encodings;
310 * do a binary search in a table of differing codepoints to see if we have one */
311 if (s <= 0 && c >= mbfl_gb18030_c_tbl_key[0] && c <= mbfl_gb18030_c_tbl_key[mbfl_gb18030_c_tbl_max-1]) {
312 k1 = mbfl_bisec_srch2(c, mbfl_gb18030_c_tbl_key, mbfl_gb18030_c_tbl_max);
313 if (k1 >= 0) {
314 s = mbfl_gb18030_c_tbl_val[k1];
315 }
316 }
317
318 if (c >= 0xe000 && c <= 0xe864) { /* PUA */
319 if (c < 0xe766) {
320 if (c < 0xe4c6) {
321 c1 = c - 0xe000;
322 s = (c1 % 94) + 0xa1;
323 c1 /= 94;
324 s |= (c1 < 0x06 ? c1 + 0xaa : c1 + 0xf2) << 8;
325 } else {
326 c1 = c - 0xe4c6;
327 s = ((c1 / 96) + 0xa1) << 8;
328 c1 %= 96;
329 s |= c1 + (c1 >= 0x3f ? 0x41 : 0x40);
330 }
331 } else {
332 /* U+E766..U+E864 */
333 k1 = 0;
334 k2 = mbfl_gb18030_pua_tbl_max;
335 while (k1 < k2) {
336 k = (k1 + k2) >> 1;
337 if (c < mbfl_gb18030_pua_tbl[k][0]) {
338 k2 = k;
339 } else if (c > mbfl_gb18030_pua_tbl[k][1]) {
340 k1 = k + 1;
341 } else {
342 s = c - mbfl_gb18030_pua_tbl[k][0] + mbfl_gb18030_pua_tbl[k][2];
343 break;
344 }
345 }
346 }
347 }
348
349 /* If we have not yet found a suitable mapping for this codepoint, it requires a 4-byte code */
350 if (s <= 0 && c >= 0x0080 && c <= 0xffff) {
351 /* BMP */
352 s = mbfl_bisec_srch(c, mbfl_uni2gb_tbl, mbfl_gb_uni_max);
353 if (s >= 0) {
354 c1 = c - mbfl_gb_uni_ofst[s];
355 s = (c1 % 10) + 0x30;
356 c1 /= 10;
357 s |= ((c1 % 126) + 0x81) << 8;
358 c1 /= 126;
359 s |= ((c1 % 10) + 0x30) << 16;
360 c1 /= 10;
361 s1 = c1 + 0x81;
362 }
363 } else if (c >= 0x10000 && c <= 0x10ffff) {
364 /* Code set 3: Unicode U+10000..U+10FFFF */
365 c1 = c - 0x10000;
366 s = (c1 % 10) + 0x30;
367 c1 /= 10;
368 s |= ((c1 % 126) + 0x81) << 8;
369 c1 /= 126;
370 s |= ((c1 % 10) + 0x30) << 16;
371 c1 /= 10;
372 s1 = c1 + 0x90;
373 }
374
375 if (s <= 0) {
376 if (c == 0) {
377 s = 0;
378 } else {
379 s = -1;
380 }
381 }
382
383 if (s >= 0) {
384 if (s <= 0x80) { /* latin */
385 CK((*filter->output_function)(s, filter->data));
386 } else if (s1 > 0) { /* qbcs */
387 CK((*filter->output_function)(s1 & 0xff, filter->data));
388 CK((*filter->output_function)((s >> 16) & 0xff, filter->data));
389 CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
390 CK((*filter->output_function)(s & 0xff, filter->data));
391 } else { /* dbcs */
392 CK((*filter->output_function)((s >> 8) & 0xff, filter->data));
393 CK((*filter->output_function)(s & 0xff, filter->data));
394 }
395 } else {
396 CK(mbfl_filt_conv_illegal_output(c, filter));
397 }
398
399 return 0;
400 }
401
