1 /*
2 * "streamable kanji code filter and converter"
3 * Copyright (c) 1998-2002 HappySize, Inc. All rights reserved.
4 *
5 * LICENSE NOTICES
6 *
7 * This file is part of "streamable kanji code filter and converter",
8 * which is distributed under the terms of GNU Lesser General Public
9 * License (version 2) as published by the Free Software Foundation.
10 *
11 * This software is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with "streamable kanji code filter and converter";
18 * if not, write to the Free Software Foundation, Inc., 59 Temple Place,
19 * Suite 330, Boston, MA 02111-1307 USA
20 *
21 * The author of this file:
22 *
23 */
24 /*
25 * The source code included in this files was separated from mbfilter.c
26 * by Moriyoshi Koizumi <moriyoshi@php.net> on 20 Dec 2002. The file
27 * mbfilter.c is included in this package .
28 *
29 */
30
31 #include <stddef.h>
32
33 #include "mbfl_encoding.h"
34 #include "mbfl_filter_output.h"
35 #include "mbfilter_pass.h"
36 #include "mbfilter_8bit.h"
37 #include "mbfilter_wchar.h"
38
39 #include "filters/mbfilter_base64.h"
40 #include "filters/mbfilter_cjk.h"
41 #include "filters/mbfilter_qprint.h"
42 #include "filters/mbfilter_uuencode.h"
43 #include "filters/mbfilter_7bit.h"
44 #include "filters/mbfilter_utf7.h"
45 #include "filters/mbfilter_utf7imap.h"
46 #include "filters/mbfilter_utf8.h"
47 #include "filters/mbfilter_utf8_mobile.h"
48 #include "filters/mbfilter_utf16.h"
49 #include "filters/mbfilter_utf32.h"
50 #include "filters/mbfilter_ucs4.h"
51 #include "filters/mbfilter_ucs2.h"
52 #include "filters/mbfilter_htmlent.h"
53 #include "filters/mbfilter_singlebyte.h"
54
55 /* hex character table "0123456789ABCDEF" */
56 static char mbfl_hexchar_table[] = {
57 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x41,0x42,0x43,0x44,0x45,0x46
58 };
59
60 static const struct mbfl_convert_vtbl *mbfl_special_filter_list[] = {
61 &vtbl_8bit_b64,
62 &vtbl_b64_8bit,
63 &vtbl_uuencode_8bit,
64 &vtbl_8bit_qprint,
65 &vtbl_qprint_8bit,
66 &vtbl_pass,
67 NULL
68 };
69
mbfl_convert_filter_init(mbfl_convert_filter * filter,const mbfl_encoding * from,const mbfl_encoding * to,const struct mbfl_convert_vtbl * vtbl,output_function_t output_function,flush_function_t flush_function,void * data)70 static void mbfl_convert_filter_init(mbfl_convert_filter *filter, const mbfl_encoding *from, const mbfl_encoding *to,
71 const struct mbfl_convert_vtbl *vtbl, output_function_t output_function, flush_function_t flush_function, void* data)
72 {
73 /* encoding structure */
74 filter->from = from;
75 filter->to = to;
76
77 if (output_function != NULL) {
78 filter->output_function = output_function;
79 } else {
80 filter->output_function = mbfl_filter_output_null;
81 }
82
83 filter->flush_function = flush_function;
84 filter->data = data;
85 filter->illegal_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR;
86 filter->illegal_substchar = '?';
87 filter->num_illegalchar = 0;
88 filter->filter_dtor = vtbl->filter_dtor;
89 filter->filter_function = vtbl->filter_function;
90 filter->filter_flush = (filter_flush_t)vtbl->filter_flush;
91 filter->filter_copy = vtbl->filter_copy;
92
93 (*vtbl->filter_ctor)(filter);
94 }
95
mbfl_convert_filter_new(const mbfl_encoding * from,const mbfl_encoding * to,output_function_t output_function,flush_function_t flush_function,void * data)96 mbfl_convert_filter* mbfl_convert_filter_new(const mbfl_encoding *from, const mbfl_encoding *to, output_function_t output_function,
97 flush_function_t flush_function, void* data)
98 {
99 const struct mbfl_convert_vtbl *vtbl = mbfl_convert_filter_get_vtbl(from, to);
100 if (vtbl == NULL) {
101 return NULL;
102 }
103
104 mbfl_convert_filter *filter = emalloc(sizeof(mbfl_convert_filter));
105 mbfl_convert_filter_init(filter, from, to, vtbl, output_function, flush_function, data);
106 return filter;
107 }
108
mbfl_convert_filter_new2(const struct mbfl_convert_vtbl * vtbl,output_function_t output_function,flush_function_t flush_function,void * data)109 mbfl_convert_filter* mbfl_convert_filter_new2(const struct mbfl_convert_vtbl *vtbl, output_function_t output_function,
110 flush_function_t flush_function, void* data)
111 {
112 const mbfl_encoding *from_encoding = mbfl_no2encoding(vtbl->from);
113 const mbfl_encoding *to_encoding = mbfl_no2encoding(vtbl->to);
114
115 mbfl_convert_filter *filter = emalloc(sizeof(mbfl_convert_filter));
116 mbfl_convert_filter_init(filter, from_encoding, to_encoding, vtbl, output_function, flush_function, data);
117 return filter;
118 }
119
mbfl_convert_filter_delete(mbfl_convert_filter * filter)120 void mbfl_convert_filter_delete(mbfl_convert_filter *filter)
121 {
122 if (filter->filter_dtor) {
123 (*filter->filter_dtor)(filter);
124 }
125 efree(filter);
126 }
127
128 /* Feed a char, return 0 if ok - used by mailparse ext */
mbfl_convert_filter_feed(int c,mbfl_convert_filter * filter)129 int mbfl_convert_filter_feed(int c, mbfl_convert_filter *filter)
130 {
131 return (*filter->filter_function)(c, filter);
132 }
133
134 /* Feed string into `filter` byte by byte; return pointer to first byte not processed */
mbfl_convert_filter_feed_string(mbfl_convert_filter * filter,unsigned char * p,size_t len)135 unsigned char* mbfl_convert_filter_feed_string(mbfl_convert_filter *filter, unsigned char *p, size_t len)
136 {
137 while (len--) {
138 if ((*filter->filter_function)(*p++, filter) < 0) {
139 break;
140 }
141 }
142 return p;
143 }
144
mbfl_convert_filter_flush(mbfl_convert_filter * filter)145 int mbfl_convert_filter_flush(mbfl_convert_filter *filter)
146 {
147 (*filter->filter_flush)(filter);
148 return 0;
149 }
150
mbfl_convert_filter_reset(mbfl_convert_filter * filter,const mbfl_encoding * from,const mbfl_encoding * to)151 void mbfl_convert_filter_reset(mbfl_convert_filter *filter, const mbfl_encoding *from, const mbfl_encoding *to)
152 {
153 if (filter->filter_dtor) {
154 (*filter->filter_dtor)(filter);
155 }
156
157 const struct mbfl_convert_vtbl *vtbl = mbfl_convert_filter_get_vtbl(from, to);
158
159 if (vtbl == NULL) {
160 vtbl = &vtbl_pass;
161 }
162
163 mbfl_convert_filter_init(filter, from, to, vtbl, filter->output_function, filter->flush_function, filter->data);
164 }
165
mbfl_convert_filter_copy(mbfl_convert_filter * src,mbfl_convert_filter * dest)166 void mbfl_convert_filter_copy(mbfl_convert_filter *src, mbfl_convert_filter *dest)
167 {
168 if (src->filter_copy != NULL) {
169 src->filter_copy(src, dest);
170 return;
171 }
172
173 *dest = *src;
174 }
175
mbfl_convert_filter_devcat(mbfl_convert_filter * filter,mbfl_memory_device * src)176 void mbfl_convert_filter_devcat(mbfl_convert_filter *filter, mbfl_memory_device *src)
177 {
178 mbfl_convert_filter_feed_string(filter, src->buffer, src->pos);
179 }
180
mbfl_convert_filter_strcat(mbfl_convert_filter * filter,const unsigned char * p)181 int mbfl_convert_filter_strcat(mbfl_convert_filter *filter, const unsigned char *p)
182 {
183 int c;
184 while ((c = *p++)) {
185 if ((*filter->filter_function)(c, filter) < 0) {
186 return -1;
187 }
188 }
189
190 return 0;
191 }
192
mbfl_filt_conv_output_hex(unsigned int w,mbfl_convert_filter * filter)193 static int mbfl_filt_conv_output_hex(unsigned int w, mbfl_convert_filter *filter)
194 {
195 bool nonzero = false;
196 int shift = 28, ret = 0;
197
198 while (shift >= 0) {
199 int n = (w >> shift) & 0xF;
200 if (n || nonzero) {
201 nonzero = true;
202 ret = (*filter->filter_function)(mbfl_hexchar_table[n], filter);
203 if (ret < 0) {
204 return ret;
205 }
206 }
207 shift -= 4;
208 }
209
210 if (!nonzero) {
211 /* No hex digits were output by above loop */
212 ret = (*filter->filter_function)('0', filter);
213 }
214
215 return ret;
216 }
217
218 /* illegal character output function for conv-filter */
mbfl_filt_conv_illegal_output(int c,mbfl_convert_filter * filter)219 int mbfl_filt_conv_illegal_output(int c, mbfl_convert_filter *filter)
220 {
221 unsigned int w = c;
222 int ret = 0;
223 int mode_backup = filter->illegal_mode;
224 uint32_t substchar_backup = filter->illegal_substchar;
225
226 /* The used substitution character may not be supported by the target character encoding.
227 * If that happens, first try to use "?" instead and if that also fails, silently drop the
228 * character. */
229 if (filter->illegal_mode == MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR
230 && filter->illegal_substchar != '?') {
231 filter->illegal_substchar = '?';
232 } else {
233 filter->illegal_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE;
234 }
235
236 switch (mode_backup) {
237 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR:
238 ret = (*filter->filter_function)(substchar_backup, filter);
239 break;
240
241 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_LONG:
242 if (w != MBFL_BAD_INPUT) {
243 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"U+");
244 if (ret < 0)
245 break;
246 ret = mbfl_filt_conv_output_hex(w, filter);
247 } else {
248 ret = (*filter->filter_function)(substchar_backup, filter);
249 }
250 break;
251
252 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_ENTITY:
253 if (w != MBFL_BAD_INPUT) {
254 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)"&#x");
255 if (ret < 0)
256 break;
257 ret = mbfl_filt_conv_output_hex(w, filter);
258 if (ret < 0)
259 break;
260 ret = mbfl_convert_filter_strcat(filter, (const unsigned char *)";");
261 } else {
262 ret = (*filter->filter_function)(substchar_backup, filter);
263 }
264 break;
265
266 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE:
267 default:
268 break;
269 }
270
271 filter->illegal_mode = mode_backup;
272 filter->illegal_substchar = substchar_backup;
273 filter->num_illegalchar++;
274
275 return ret;
276 }
277
mbfl_convert_filter_get_vtbl(const mbfl_encoding * from,const mbfl_encoding * to)278 const struct mbfl_convert_vtbl* mbfl_convert_filter_get_vtbl(const mbfl_encoding *from, const mbfl_encoding *to)
279 {
280 if (to->no_encoding == mbfl_no_encoding_base64 ||
281 to->no_encoding == mbfl_no_encoding_qprint) {
282 from = &mbfl_encoding_8bit;
283 } else if (from->no_encoding == mbfl_no_encoding_base64 ||
284 from->no_encoding == mbfl_no_encoding_qprint ||
285 from->no_encoding == mbfl_no_encoding_uuencode) {
286 to = &mbfl_encoding_8bit;
287 }
288
289 if (to == from && (to == &mbfl_encoding_wchar || to == &mbfl_encoding_8bit)) {
290 return &vtbl_pass;
291 }
292
293 if (to->no_encoding == mbfl_no_encoding_wchar) {
294 return from->input_filter;
295 } else if (from->no_encoding == mbfl_no_encoding_wchar) {
296 return to->output_filter;
297 } else {
298 int i = 0;
299 const struct mbfl_convert_vtbl *vtbl;
300 while ((vtbl = mbfl_special_filter_list[i++])) {
301 if (vtbl->from == from->no_encoding && vtbl->to == to->no_encoding) {
302 return vtbl;
303 }
304 }
305 return NULL;
306 }
307 }
308
309 /*
310 * commonly used constructor
311 */
mbfl_filt_conv_common_ctor(mbfl_convert_filter * filter)312 void mbfl_filt_conv_common_ctor(mbfl_convert_filter *filter)
313 {
314 filter->status = filter->cache = 0;
315 }
316
mbfl_filt_conv_common_flush(mbfl_convert_filter * filter)317 int mbfl_filt_conv_common_flush(mbfl_convert_filter *filter)
318 {
319 if (filter->flush_function) {
320 (*filter->flush_function)(filter->data);
321 }
322 return 0;
323 }
324
mb_fast_convert(unsigned char * in,size_t in_len,const mbfl_encoding * from,const mbfl_encoding * to,uint32_t replacement_char,unsigned int error_mode,unsigned int * num_errors)325 zend_string* mb_fast_convert(unsigned char *in, size_t in_len, const mbfl_encoding *from, const mbfl_encoding *to, uint32_t replacement_char, unsigned int error_mode, unsigned int *num_errors)
326 {
327 uint32_t wchar_buf[128];
328 unsigned int state = 0;
329
330 if (to == &mbfl_encoding_base64 || to == &mbfl_encoding_qprint) {
331 from = &mbfl_encoding_8bit;
332 } else if (from == &mbfl_encoding_base64 || from == &mbfl_encoding_qprint || from == &mbfl_encoding_uuencode) {
333 to = &mbfl_encoding_8bit;
334 }
335
336 mb_convert_buf buf;
337 mb_convert_buf_init(&buf, in_len, replacement_char, error_mode);
338
339 while (in_len) {
340 size_t out_len = from->to_wchar(&in, &in_len, wchar_buf, 128, &state);
341 ZEND_ASSERT(out_len <= 128);
342 to->from_wchar(wchar_buf, out_len, &buf, !in_len);
343 }
344
345 *num_errors = buf.errors;
346 return mb_convert_buf_result(&buf, to);
347 }
348
convert_cp_to_hex(uint32_t cp,uint32_t * out)349 static uint32_t* convert_cp_to_hex(uint32_t cp, uint32_t *out)
350 {
351 bool nonzero = false;
352 int shift = 28;
353
354 while (shift >= 0) {
355 int n = (cp >> shift) & 0xF;
356 if (n || nonzero) {
357 nonzero = true;
358 *out++ = mbfl_hexchar_table[n];
359 }
360 shift -= 4;
361 }
362
363 if (!nonzero) {
364 /* No hex digits were output by above loop */
365 *out++ = '0';
366 }
367
368 return out;
369 }
370
mb_illegal_marker(uint32_t bad_cp,uint32_t * out,unsigned int err_mode,uint32_t replacement_char)371 static size_t mb_illegal_marker(uint32_t bad_cp, uint32_t *out, unsigned int err_mode, uint32_t replacement_char)
372 {
373 uint32_t *start = out;
374
375 if (bad_cp == MBFL_BAD_INPUT) {
376 /* Input string contained a byte sequence which was invalid in the 'from' encoding
377 * Unless the error handling mode is set to NONE, insert the replacement character */
378 if (err_mode != MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE) {
379 *out++ = replacement_char;
380 }
381 } else {
382 /* Input string contained a byte sequence which was valid in the 'from' encoding,
383 * but decoded to a Unicode codepoint which cannot be represented in the 'to' encoding */
384 switch (err_mode) {
385 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR:
386 *out++ = replacement_char;
387 break;
388
389 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_LONG:
390 out[0] = 'U';
391 out[1] = '+';
392 out = convert_cp_to_hex(bad_cp, &out[2]);
393 break;
394
395 case MBFL_OUTPUTFILTER_ILLEGAL_MODE_ENTITY:
396 out[0] = '&'; out[1] = '#'; out[2] = 'x';
397 out = convert_cp_to_hex(bad_cp, &out[3]);
398 *out++ = ';';
399 break;
400 }
401 }
402
403 return out - start;
404 }
405
mb_illegal_output(uint32_t bad_cp,mb_from_wchar_fn fn,mb_convert_buf * buf)406 void mb_illegal_output(uint32_t bad_cp, mb_from_wchar_fn fn, mb_convert_buf* buf)
407 {
408 buf->errors++;
409
410 uint32_t temp[12];
411 uint32_t repl_char = buf->replacement_char;
412 unsigned int err_mode = buf->error_mode;
413
414 if (err_mode == MBFL_OUTPUTFILTER_ILLEGAL_MODE_BADUTF8) {
415 /* This mode is for internal use only, when converting a string to
416 * UTF-8 before searching it; it uses a byte which is illegal in
417 * UTF-8 as an error marker. This ensures that error markers will
418 * never 'accidentally' match valid text, as could happen when a
419 * character like '?' is used as an error marker. */
420 MB_CONVERT_BUF_ENSURE(buf, buf->out, buf->limit, 1);
421 buf->out = mb_convert_buf_add(buf->out, 0xFF);
422 return;
423 }
424
425 size_t len = mb_illegal_marker(bad_cp, temp, err_mode, repl_char);
426
427 /* Avoid infinite loop if `fn` is not able to handle `repl_char` */
428 if (err_mode == MBFL_OUTPUTFILTER_ILLEGAL_MODE_CHAR && repl_char != '?') {
429 buf->replacement_char = '?';
430 } else {
431 buf->error_mode = MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE;
432 }
433
434 fn(temp, len, buf, false);
435
436 buf->replacement_char = repl_char;
437 buf->error_mode = err_mode;
438 }
439