1 /*
2 * "streamable kanji code filter and converter"
3 * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved.
4 *
5 * LICENSE NOTICES
6 *
7 * This file is part of "streamable kanji code filter and converter",
8 * which is distributed under the terms of GNU Lesser General Public
9 * License (version 2) as published by the Free Software Foundation.
10 *
11 * This software is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with "streamable kanji code filter and converter";
18 * if not, write to the Free Software Foundation, Inc., 59 Temple Place,
19 * Suite 330, Boston, MA 02111-1307 USA
20 *
21 * The author of this file:
22 *
23 */
24 /*
25 * The source code included in this files was separated from mbfilter.c
26 * by Moriyoshi Koizumi <moriyoshi@php.net> on 20 Dec 2002. The file
27 * mbfilter.c is included in this package .
28 *
29 */
30
31 #include <stddef.h>
32
33 #include "mbfl_encoding.h"
34 #include "mbfl_filter_output.h"
35 #include "mbfilter_pass.h"
36 #include "mbfilter_8bit.h"
37 #include "mbfilter_wchar.h"
38
39 #include "filters/mbfilter_euc_cn.h"
40 #include "filters/mbfilter_hz.h"
41 #include "filters/mbfilter_euc_tw.h"
42 #include "filters/mbfilter_big5.h"
43 #include "filters/mbfilter_uhc.h"
44 #include "filters/mbfilter_euc_kr.h"
45 #include "filters/mbfilter_iso2022_kr.h"
46 #include "filters/mbfilter_sjis.h"
47 #include "filters/mbfilter_sjis_open.h"
48 #include "filters/mbfilter_sjis_2004.h"
49 #include "filters/mbfilter_sjis_mobile.h"
50 #include "filters/mbfilter_sjis_mac.h"
51 #include "filters/mbfilter_cp51932.h"
52 #include "filters/mbfilter_jis.h"
53 #include "filters/mbfilter_iso2022_jp_ms.h"
54 #include "filters/mbfilter_iso2022jp_2004.h"
55 #include "filters/mbfilter_iso2022jp_mobile.h"
56 #include "filters/mbfilter_euc_jp.h"
57 #include "filters/mbfilter_euc_jp_2004.h"
58 #include "filters/mbfilter_euc_jp_win.h"
59 #include "filters/mbfilter_gb18030.h"
60 #include "filters/mbfilter_ascii.h"
61 #include "filters/mbfilter_koi8r.h"
62 #include "filters/mbfilter_koi8u.h"
63 #include "filters/mbfilter_cp866.h"
64 #include "filters/mbfilter_cp932.h"
65 #include "filters/mbfilter_cp936.h"
66 #include "filters/mbfilter_cp1251.h"
67 #include "filters/mbfilter_cp1252.h"
68 #include "filters/mbfilter_cp1254.h"
69 #include "filters/mbfilter_cp5022x.h"
70 #include "filters/mbfilter_iso8859_1.h"
71 #include "filters/mbfilter_iso8859_2.h"
72 #include "filters/mbfilter_iso8859_3.h"
73 #include "filters/mbfilter_iso8859_4.h"
74 #include "filters/mbfilter_iso8859_5.h"
75 #include "filters/mbfilter_iso8859_6.h"
76 #include "filters/mbfilter_iso8859_7.h"
77 #include "filters/mbfilter_iso8859_8.h"
78 #include "filters/mbfilter_iso8859_9.h"
79 #include "filters/mbfilter_iso8859_10.h"
80 #include "filters/mbfilter_iso8859_13.h"
81 #include "filters/mbfilter_iso8859_14.h"
82 #include "filters/mbfilter_iso8859_15.h"
83 #include "filters/mbfilter_base64.h"
84 #include "filters/mbfilter_qprint.h"
85 #include "filters/mbfilter_uuencode.h"
86 #include "filters/mbfilter_7bit.h"
87 #include "filters/mbfilter_utf7.h"
88 #include "filters/mbfilter_utf7imap.h"
89 #include "filters/mbfilter_utf8.h"
90 #include "filters/mbfilter_utf8_mobile.h"
91 #include "filters/mbfilter_utf16.h"
92 #include "filters/mbfilter_utf32.h"
93 #include "filters/mbfilter_byte2.h"
94 #include "filters/mbfilter_byte4.h"
95 #include "filters/mbfilter_ucs4.h"
96 #include "filters/mbfilter_ucs2.h"
97 #include "filters/mbfilter_htmlent.h"
98 #include "filters/mbfilter_armscii8.h"
99 #include "filters/mbfilter_cp850.h"
100
101 /* hex character table "0123456789ABCDEF" */
102 static char mbfl_hexchar_table[] = {
103 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x41,0x42,0x43,0x44,0x45,0x46
104 };
105
106 static const struct mbfl_convert_vtbl *mbfl_special_filter_list[] = {
107 &vtbl_8bit_b64,
108 &vtbl_b64_8bit,
109 &vtbl_uuencode_8bit,
110 &vtbl_8bit_qprint,
111 &vtbl_qprint_8bit,
112 &vtbl_8bit_7bit,
113 &vtbl_7bit_8bit,
114 &vtbl_pass,
115 NULL
116 };
117
mbfl_convert_filter_common_init(mbfl_convert_filter * filter,const mbfl_encoding * from,const mbfl_encoding * to,const struct mbfl_convert_vtbl * vtbl,output_function_t output_function,flush_function_t flush_function,void * data)118 static void mbfl_convert_filter_common_init(mbfl_convert_filter *filter, const mbfl_encoding *from, const mbfl_encoding *to,
119 const struct mbfl_convert_vtbl *vtbl, output_function_t output_function, flush_function_t flush_function, void* data)
120 {
121 /* encoding structure */
122 filter->from = from;
123 filter->to = to;
124
125 if (output_function != NULL) {
126 filter->output_function = output_function;
127 } else {
128 filter->output_function = mbfl_filter_output_null;
129 }
130
131 filter->flush_function = flush_function;
132 filter->data = data;
133 filter->illegal_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR;
134 filter->illegal_substchar = '?';
135 filter->num_illegalchar = 0;
136 filter->filter_ctor = vtbl->filter_ctor;
137 filter->filter_dtor = vtbl->filter_dtor;
138 filter->filter_function = vtbl->filter_function;
139 filter->filter_flush = vtbl->filter_flush;
140 filter->filter_copy = vtbl->filter_copy;
141
142 (*filter->filter_ctor)(filter);
143 }
144
145
mbfl_convert_filter_new(const mbfl_encoding * from,const mbfl_encoding * to,output_function_t output_function,flush_function_t flush_function,void * data)146 mbfl_convert_filter* mbfl_convert_filter_new(const mbfl_encoding *from, const mbfl_encoding *to, output_function_t output_function,
147 flush_function_t flush_function, void* data)
148 {
149 const struct mbfl_convert_vtbl *vtbl = mbfl_convert_filter_get_vtbl(from, to);
150 if (vtbl == NULL) {
151 return NULL;
152 }
153
154 mbfl_convert_filter *filter = emalloc(sizeof(mbfl_convert_filter));
155 mbfl_convert_filter_common_init(filter, from, to, vtbl, output_function, flush_function, data);
156 return filter;
157 }
158
mbfl_convert_filter_new2(const struct mbfl_convert_vtbl * vtbl,output_function_t output_function,flush_function_t flush_function,void * data)159 mbfl_convert_filter* mbfl_convert_filter_new2(const struct mbfl_convert_vtbl *vtbl, output_function_t output_function,
160 flush_function_t flush_function, void* data)
161 {
162 const mbfl_encoding *from_encoding = mbfl_no2encoding(vtbl->from);
163 const mbfl_encoding *to_encoding = mbfl_no2encoding(vtbl->to);
164
165 mbfl_convert_filter *filter = emalloc(sizeof(mbfl_convert_filter));
166 mbfl_convert_filter_common_init(filter, from_encoding, to_encoding, vtbl, output_function, flush_function, data);
167 return filter;
168 }
169
mbfl_convert_filter_delete(mbfl_convert_filter * filter)170 void mbfl_convert_filter_delete(mbfl_convert_filter *filter)
171 {
172 if (filter->filter_dtor) {
173 (*filter->filter_dtor)(filter);
174 }
175 efree(filter);
176 }
177
178 /* Feed a char, return 0 if ok - used by mailparse ext */
mbfl_convert_filter_feed(int c,mbfl_convert_filter * filter)179 int mbfl_convert_filter_feed(int c, mbfl_convert_filter *filter)
180 {
181 return (*filter->filter_function)(c, filter);
182 }
183
184 /* Feed string into `filter` byte by byte; return pointer to first byte not processed */
mbfl_convert_filter_feed_string(mbfl_convert_filter * filter,unsigned char * p,size_t len)185 unsigned char* mbfl_convert_filter_feed_string(mbfl_convert_filter *filter, unsigned char *p, size_t len)
186 {
187 while (len--) {
188 if ((*filter->filter_function)(*p++, filter) < 0) {
189 break;
190 }
191 }
192 return p;
193 }
194
mbfl_convert_filter_flush(mbfl_convert_filter * filter)195 int mbfl_convert_filter_flush(mbfl_convert_filter *filter)
196 {
197 (*filter->filter_flush)(filter);
198 return filter->flush_function ? (*filter->flush_function)(filter->data) : 0;
199 }
200
mbfl_convert_filter_reset(mbfl_convert_filter * filter,const mbfl_encoding * from,const mbfl_encoding * to)201 void mbfl_convert_filter_reset(mbfl_convert_filter *filter, const mbfl_encoding *from, const mbfl_encoding *to)
202 {
203 if (filter->filter_dtor) {
204 (*filter->filter_dtor)(filter);
205 }
206
207 const struct mbfl_convert_vtbl *vtbl = mbfl_convert_filter_get_vtbl(from, to);
208
209 if (vtbl == NULL) {
210 vtbl = &vtbl_pass;
211 }
212
213 mbfl_convert_filter_common_init(filter, from, to, vtbl,
214 filter->output_function, filter->flush_function, filter->data);
215 }
216
mbfl_convert_filter_copy(mbfl_convert_filter * src,mbfl_convert_filter * dest)217 void mbfl_convert_filter_copy(mbfl_convert_filter *src, mbfl_convert_filter *dest)
218 {
219 if (src->filter_copy != NULL) {
220 src->filter_copy(src, dest);
221 return;
222 }
223
224 *dest = *src;
225 }
226
mbfl_convert_filter_devcat(mbfl_convert_filter * filter,mbfl_memory_device * src)227 void mbfl_convert_filter_devcat(mbfl_convert_filter *filter, mbfl_memory_device *src)
228 {
229 mbfl_convert_filter_feed_string(filter, src->buffer, src->pos);
230 }
231
mbfl_convert_filter_strcat(mbfl_convert_filter * filter,const unsigned char * p)232 int mbfl_convert_filter_strcat(mbfl_convert_filter *filter, const unsigned char *p)
233 {
234 int c;
235 while ((c = *p++)) {
236 if ((*filter->filter_function)(c, filter) < 0) {
237 return -1;
238 }
239 }
240
241 return 0;
242 }
243
244 /* illegal character output function for conv-filter */
mbfl_filt_conv_illegal_output(int c,mbfl_convert_filter * filter)245 int mbfl_filt_conv_illegal_output(int c, mbfl_convert_filter *filter)
246 {
247 int n, m, r;
248
249 int ret = 0;
250 int mode_backup = filter->illegal_mode;
251 int substchar_backup = filter->illegal_substchar;
252
253 /* The used substitution character may not be supported by the target character encoding.
254 * If that happens, first try to use "?" instead and if that also fails, silently drop the
255 * character. */
256 if (filter->illegal_mode == MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR
257 && filter->illegal_substchar != '?') {
258 filter->illegal_substchar = '?';
259 } else {
260 filter->illegal_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE;
261 }
262
263 switch (mode_backup) {
264 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR:
265 ret = (*filter->filter_function)(substchar_backup, filter);
266 break;
267 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_LONG:
268 if (c >= 0) {
269 if (c < MBFL_WCSGROUP_UCS4MAX) { /* unicode */
270 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"U+");
271 } else {
272 if (c < MBFL_WCSGROUP_WCHARMAX) {
273 m = c & ~MBFL_WCSPLANE_MASK;
274 switch (m) {
275 case MBFL_WCSPLANE_JIS0208:
276 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"JIS+");
277 break;
278 case MBFL_WCSPLANE_JIS0212:
279 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"JIS2+");
280 break;
281 case MBFL_WCSPLANE_JIS0213:
282 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"JIS3+");
283 break;
284 case MBFL_WCSPLANE_WINCP932:
285 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"W932+");
286 break;
287 case MBFL_WCSPLANE_GB18030:
288 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"GB+");
289 break;
290 case MBFL_WCSPLANE_8859_1:
291 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"I8859_1+");
292 break;
293 default:
294 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"?+");
295 break;
296 }
297 c &= MBFL_WCSPLANE_MASK;
298 } else {
299 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"BAD+");
300 c &= MBFL_WCSGROUP_MASK;
301 }
302 }
303 if (ret >= 0) {
304 m = 0;
305 r = 28;
306 while (r >= 0) {
307 n = (c >> r) & 0xf;
308 if (n || m) {
309 m = 1;
310 ret = (*filter->filter_function)(mbfl_hexchar_table[n], filter);
311 if (ret < 0) {
312 break;
313 }
314 }
315 r -= 4;
316 }
317 if (m == 0) {
318 ret = (*filter->filter_function)(mbfl_hexchar_table[0], filter);
319 }
320 }
321 }
322 break;
323 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_ENTITY:
324 if (c >= 0) {
325 if (c < MBFL_WCSGROUP_UCS4MAX) { /* unicode */
326 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"&#x");
327 if (ret < 0)
328 break;
329
330 m = 0;
331 r = 28;
332 while (r >= 0) {
333 n = (c >> r) & 0xf;
334 if (n || m) {
335 m = 1;
336 ret = (*filter->filter_function)(mbfl_hexchar_table[n], filter);
337 if (ret < 0) {
338 break;
339 }
340 }
341 r -= 4;
342 }
343 if (m == 0) {
344 /* illegal character was zero; no hex digits were output by above loop */
345 ret = (*filter->filter_function)('0', filter);
346 }
347 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)";");
348 } else {
349 ret = (*filter->filter_function)(substchar_backup, filter);
350 }
351 }
352 break;
353 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE:
354 default:
355 break;
356 }
357
358 filter->illegal_mode = mode_backup;
359 filter->illegal_substchar = substchar_backup;
360 filter->num_illegalchar++;
361
362 return ret;
363 }
364
mbfl_convert_filter_get_vtbl(const mbfl_encoding * from,const mbfl_encoding * to)365 const struct mbfl_convert_vtbl* mbfl_convert_filter_get_vtbl(const mbfl_encoding *from, const mbfl_encoding *to)
366 {
367 if (to->no_encoding == mbfl_no_encoding_base64 ||
368 to->no_encoding == mbfl_no_encoding_qprint ||
369 to->no_encoding == mbfl_no_encoding_7bit) {
370 from = &mbfl_encoding_8bit;
371 } else if (from->no_encoding == mbfl_no_encoding_base64 ||
372 from->no_encoding == mbfl_no_encoding_qprint ||
373 from->no_encoding == mbfl_no_encoding_uuencode) {
374 to = &mbfl_encoding_8bit;
375 }
376
377 if (to == from && (to == &mbfl_encoding_wchar || to == &mbfl_encoding_8bit)) {
378 return &vtbl_pass;
379 }
380
381 if (to->no_encoding == mbfl_no_encoding_wchar) {
382 return from->input_filter;
383 } else if (from->no_encoding == mbfl_no_encoding_wchar) {
384 return to->output_filter;
385 } else {
386 int i = 0;
387 const struct mbfl_convert_vtbl *vtbl;
388 while ((vtbl = mbfl_special_filter_list[i++])) {
389 if (vtbl->from == from->no_encoding && vtbl->to == to->no_encoding) {
390 return vtbl;
391 }
392 }
393 return NULL;
394 }
395 }
396
397 /*
398 * commonly used constructor
399 */
mbfl_filt_conv_common_ctor(mbfl_convert_filter * filter)400 void mbfl_filt_conv_common_ctor(mbfl_convert_filter *filter)
401 {
402 filter->status = 0;
403 filter->cache = 0;
404 }
405
mbfl_filt_conv_common_flush(mbfl_convert_filter * filter)406 int mbfl_filt_conv_common_flush(mbfl_convert_filter *filter)
407 {
408 filter->status = 0;
409 filter->cache = 0;
410
411 if (filter->flush_function != NULL) {
412 (*filter->flush_function)(filter->data);
413 }
414 return 0;
415 }
416
