xref: /PHP-8.0/ext/fileinfo/libmagic/encoding.c (revision fa96c7e0)
1 /*
2  * Copyright (c) Ian F. Darwin 1986-1995.
3  * Software written by Ian F. Darwin and others;
4  * maintained 1995-present by Christos Zoulas and others.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice immediately at the beginning of the file, without modification,
11  *    this list of conditions, and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 /*
29  * Encoding -- determine the character encoding of a text file.
30  *
31  * Joerg Wunsch <joerg@freebsd.org> wrote the original support for 8-bit
32  * international characters.
33  */
34 
35 #include "file.h"
36 
37 #ifndef	lint
38 FILE_RCSID("@(#)$File: encoding.c,v 1.21 2019/06/08 20:49:14 christos Exp $")
39 #endif	/* lint */
40 
41 #include "magic.h"
42 #include <string.h>
43 #include <stdlib.h>
44 
45 
46 private int looks_ascii(const unsigned char *, size_t, unicodechar *, size_t *);
47 private int looks_utf8_with_BOM(const unsigned char *, size_t, unicodechar *,
48     size_t *);
49 private int looks_utf7(const unsigned char *, size_t, unicodechar *, size_t *);
50 private int looks_ucs16(const unsigned char *, size_t, unicodechar *, size_t *);
51 private int looks_ucs32(const unsigned char *, size_t, unicodechar *, size_t *);
52 private int looks_latin1(const unsigned char *, size_t, unicodechar *, size_t *);
53 private int looks_extended(const unsigned char *, size_t, unicodechar *, size_t *);
54 private void from_ebcdic(const unsigned char *, size_t, unsigned char *);
55 
56 #ifdef DEBUG_ENCODING
57 #define DPRINTF(a) printf a
58 #else
59 #define DPRINTF(a)
60 #endif
61 
62 /*
63  * Try to determine whether text is in some character code we can
64  * identify.  Each of these tests, if it succeeds, will leave
65  * the text converted into one-unicodechar-per-character Unicode in
66  * ubuf, and the number of characters converted in ulen.
67  */
68 protected int
file_encoding(struct magic_set * ms,const struct buffer * b,unicodechar ** ubuf,size_t * ulen,const char ** code,const char ** code_mime,const char ** type)69 file_encoding(struct magic_set *ms, const struct buffer *b, unicodechar **ubuf,
70     size_t *ulen, const char **code, const char **code_mime, const char **type)
71 {
72 	const unsigned char *buf = CAST(const unsigned char *, b->fbuf);
73 	size_t nbytes = b->flen > 64*1024 ? 64*1024 : b->flen;
74 	size_t mlen;
75 	int rv = 1, ucs_type;
76 	unsigned char *nbuf = NULL;
77 	unicodechar *udefbuf;
78 	size_t udeflen;
79 
80 	if (ubuf == NULL)
81 		ubuf = &udefbuf;
82 	if (ulen == NULL)
83 		ulen = &udeflen;
84 
85 	*type = "text";
86 	*ulen = 0;
87 	*code = "unknown";
88 	*code_mime = "binary";
89 
90 	mlen = (nbytes + 1) * sizeof((*ubuf)[0]);
91 	if ((*ubuf = CAST(unicodechar *, ecalloc(CAST(size_t, 1), mlen))) == NULL) {
92 		file_oomem(ms, mlen);
93 		goto done;
94 	}
95 	mlen = (nbytes + 1) * sizeof(nbuf[0]);
96 	if ((nbuf = CAST(unsigned char *,
97 	    ecalloc(CAST(size_t, 1), mlen))) == NULL) {
98 		file_oomem(ms, mlen);
99 		goto done;
100 	}
101 
102 	if (looks_ascii(buf, nbytes, *ubuf, ulen)) {
103 		if (looks_utf7(buf, nbytes, *ubuf, ulen) > 0) {
104 			DPRINTF(("utf-7 %" SIZE_T_FORMAT "u\n", *ulen));
105 			*code = "UTF-7 Unicode";
106 			*code_mime = "utf-7";
107 		} else {
108 			DPRINTF(("ascii %" SIZE_T_FORMAT "u\n", *ulen));
109 			*code = "ASCII";
110 			*code_mime = "us-ascii";
111 		}
112 	} else if (looks_utf8_with_BOM(buf, nbytes, *ubuf, ulen) > 0) {
113 		DPRINTF(("utf8/bom %" SIZE_T_FORMAT "u\n", *ulen));
114 		*code = "UTF-8 Unicode (with BOM)";
115 		*code_mime = "utf-8";
116 	} else if (file_looks_utf8(buf, nbytes, *ubuf, ulen) > 1) {
117 		DPRINTF(("utf8 %" SIZE_T_FORMAT "u\n", *ulen));
118 		*code = "UTF-8 Unicode";
119 		*code_mime = "utf-8";
120 	} else if ((ucs_type = looks_ucs32(buf, nbytes, *ubuf, ulen)) != 0) {
121 		if (ucs_type == 1) {
122 			*code = "Little-endian UTF-32 Unicode";
123 			*code_mime = "utf-32le";
124 		} else {
125 			*code = "Big-endian UTF-32 Unicode";
126 			*code_mime = "utf-32be";
127 		}
128 		DPRINTF(("ucs32 %" SIZE_T_FORMAT "u\n", *ulen));
129 	} else if ((ucs_type = looks_ucs16(buf, nbytes, *ubuf, ulen)) != 0) {
130 		if (ucs_type == 1) {
131 			*code = "Little-endian UTF-16 Unicode";
132 			*code_mime = "utf-16le";
133 		} else {
134 			*code = "Big-endian UTF-16 Unicode";
135 			*code_mime = "utf-16be";
136 		}
137 		DPRINTF(("ucs16 %" SIZE_T_FORMAT "u\n", *ulen));
138 	} else if (looks_latin1(buf, nbytes, *ubuf, ulen)) {
139 		DPRINTF(("latin1 %" SIZE_T_FORMAT "u\n", *ulen));
140 		*code = "ISO-8859";
141 		*code_mime = "iso-8859-1";
142 	} else if (looks_extended(buf, nbytes, *ubuf, ulen)) {
143 		DPRINTF(("extended %" SIZE_T_FORMAT "u\n", *ulen));
144 		*code = "Non-ISO extended-ASCII";
145 		*code_mime = "unknown-8bit";
146 	} else {
147 		from_ebcdic(buf, nbytes, nbuf);
148 
149 		if (looks_ascii(nbuf, nbytes, *ubuf, ulen)) {
150 			DPRINTF(("ebcdic %" SIZE_T_FORMAT "u\n", *ulen));
151 			*code = "EBCDIC";
152 			*code_mime = "ebcdic";
153 		} else if (looks_latin1(nbuf, nbytes, *ubuf, ulen)) {
154 			DPRINTF(("ebcdic/international %" SIZE_T_FORMAT "u\n",
155 			    *ulen));
156 			*code = "International EBCDIC";
157 			*code_mime = "ebcdic";
158 		} else { /* Doesn't look like text at all */
159 			DPRINTF(("binary\n"));
160 			rv = 0;
161 			*type = "binary";
162 		}
163 	}
164 
165  done:
166 	efree(nbuf);
167 	if (ubuf == &udefbuf)
168 		efree(udefbuf);
169 
170 	return rv;
171 }
172 
173 /*
174  * This table reflects a particular philosophy about what constitutes
175  * "text," and there is room for disagreement about it.
176  *
177  * Version 3.31 of the file command considered a file to be ASCII if
178  * each of its characters was approved by either the isascii() or
179  * isalpha() function.  On most systems, this would mean that any
180  * file consisting only of characters in the range 0x00 ... 0x7F
181  * would be called ASCII text, but many systems might reasonably
182  * consider some characters outside this range to be alphabetic,
183  * so the file command would call such characters ASCII.  It might
184  * have been more accurate to call this "considered textual on the
185  * local system" than "ASCII."
186  *
187  * It considered a file to be "International language text" if each
188  * of its characters was either an ASCII printing character (according
189  * to the real ASCII standard, not the above test), a character in
190  * the range 0x80 ... 0xFF, or one of the following control characters:
191  * backspace, tab, line feed, vertical tab, form feed, carriage return,
192  * escape.  No attempt was made to determine the language in which files
193  * of this type were written.
194  *
195  *
196  * The table below considers a file to be ASCII if all of its characters
197  * are either ASCII printing characters (again, according to the X3.4
198  * standard, not isascii()) or any of the following controls: bell,
199  * backspace, tab, line feed, form feed, carriage return, esc, nextline.
200  *
201  * I include bell because some programs (particularly shell scripts)
202  * use it literally, even though it is rare in normal text.  I exclude
203  * vertical tab because it never seems to be used in real text.  I also
204  * include, with hesitation, the X3.64/ECMA-43 control nextline (0x85),
205  * because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline
206  * character to.  It might be more appropriate to include it in the 8859
207  * set instead of the ASCII set, but it's got to be included in *something*
208  * we recognize or EBCDIC files aren't going to be considered textual.
209  * Some old Unix source files use SO/SI (^N/^O) to shift between Greek
210  * and Latin characters, so these should possibly be allowed.  But they
211  * make a real mess on VT100-style displays if they're not paired properly,
212  * so we are probably better off not calling them text.
213  *
214  * A file is considered to be ISO-8859 text if its characters are all
215  * either ASCII, according to the above definition, or printing characters
216  * from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF.
217  *
218  * Finally, a file is considered to be international text from some other
219  * character code if its characters are all either ISO-8859 (according to
220  * the above definition) or characters in the range 0x80 ... 0x9F, which
221  * ISO-8859 considers to be control characters but the IBM PC and Macintosh
222  * consider to be printing characters.
223  */
224 
225 #define F 0   /* character never appears in text */
226 #define T 1   /* character appears in plain ASCII text */
227 #define I 2   /* character appears in ISO-8859 text */
228 #define X 3   /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
229 
230 private char text_chars[256] = {
231 	/*                  BEL BS HT LF VT FF CR    */
232 	F, F, F, F, F, F, F, T, T, T, T, T, T, T, F, F,  /* 0x0X */
233 	/*                              ESC          */
234 	F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F,  /* 0x1X */
235 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x2X */
236 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x3X */
237 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x4X */
238 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x5X */
239 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T,  /* 0x6X */
240 	T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F,  /* 0x7X */
241 	/*            NEL                            */
242 	X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X,  /* 0x8X */
243 	X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X,  /* 0x9X */
244 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xaX */
245 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xbX */
246 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xcX */
247 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xdX */
248 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I,  /* 0xeX */
249 	I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I   /* 0xfX */
250 };
251 
252 private int
looks_ascii(const unsigned char * buf,size_t nbytes,unicodechar * ubuf,size_t * ulen)253 looks_ascii(const unsigned char *buf, size_t nbytes, unicodechar *ubuf,
254     size_t *ulen)
255 {
256 	size_t i;
257 
258 	*ulen = 0;
259 
260 	for (i = 0; i < nbytes; i++) {
261 		int t = text_chars[buf[i]];
262 
263 		if (t != T)
264 			return 0;
265 
266 		ubuf[(*ulen)++] = buf[i];
267 	}
268 
269 	return 1;
270 }
271 
272 private int
looks_latin1(const unsigned char * buf,size_t nbytes,unicodechar * ubuf,size_t * ulen)273 looks_latin1(const unsigned char *buf, size_t nbytes, unicodechar *ubuf, size_t *ulen)
274 {
275 	size_t i;
276 
277 	*ulen = 0;
278 
279 	for (i = 0; i < nbytes; i++) {
280 		int t = text_chars[buf[i]];
281 
282 		if (t != T && t != I)
283 			return 0;
284 
285 		ubuf[(*ulen)++] = buf[i];
286 	}
287 
288 	return 1;
289 }
290 
291 private int
looks_extended(const unsigned char * buf,size_t nbytes,unicodechar * ubuf,size_t * ulen)292 looks_extended(const unsigned char *buf, size_t nbytes, unicodechar *ubuf,
293     size_t *ulen)
294 {
295 	size_t i;
296 
297 	*ulen = 0;
298 
299 	for (i = 0; i < nbytes; i++) {
300 		int t = text_chars[buf[i]];
301 
302 		if (t != T && t != I && t != X)
303 			return 0;
304 
305 		ubuf[(*ulen)++] = buf[i];
306 	}
307 
308 	return 1;
309 }
310 
311 /*
312  * Decide whether some text looks like UTF-8. Returns:
313  *
314  *     -1: invalid UTF-8
315  *      0: uses odd control characters, so doesn't look like text
316  *      1: 7-bit text
317  *      2: definitely UTF-8 text (valid high-bit set bytes)
318  *
319  * If ubuf is non-NULL on entry, text is decoded into ubuf, *ulen;
320  * ubuf must be big enough!
321  */
322 protected int
file_looks_utf8(const unsigned char * buf,size_t nbytes,unicodechar * ubuf,size_t * ulen)323 file_looks_utf8(const unsigned char *buf, size_t nbytes, unicodechar *ubuf, size_t *ulen)
324 {
325 	size_t i;
326 	int n;
327 	unicodechar c;
328 	int gotone = 0, ctrl = 0;
329 
330 	if (ubuf)
331 		*ulen = 0;
332 
333 	for (i = 0; i < nbytes; i++) {
334 		if ((buf[i] & 0x80) == 0) {	   /* 0xxxxxxx is plain ASCII */
335 			/*
336 			 * Even if the whole file is valid UTF-8 sequences,
337 			 * still reject it if it uses weird control characters.
338 			 */
339 
340 			if (text_chars[buf[i]] != T)
341 				ctrl = 1;
342 
343 			if (ubuf)
344 				ubuf[(*ulen)++] = buf[i];
345 		} else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
346 			return -1;
347 		} else {			   /* 11xxxxxx begins UTF-8 */
348 			int following;
349 
350 			if ((buf[i] & 0x20) == 0) {		/* 110xxxxx */
351 				c = buf[i] & 0x1f;
352 				following = 1;
353 			} else if ((buf[i] & 0x10) == 0) {	/* 1110xxxx */
354 				c = buf[i] & 0x0f;
355 				following = 2;
356 			} else if ((buf[i] & 0x08) == 0) {	/* 11110xxx */
357 				c = buf[i] & 0x07;
358 				following = 3;
359 			} else if ((buf[i] & 0x04) == 0) {	/* 111110xx */
360 				c = buf[i] & 0x03;
361 				following = 4;
362 			} else if ((buf[i] & 0x02) == 0) {	/* 1111110x */
363 				c = buf[i] & 0x01;
364 				following = 5;
365 			} else
366 				return -1;
367 
368 			for (n = 0; n < following; n++) {
369 				i++;
370 				if (i >= nbytes)
371 					goto done;
372 
373 				if ((buf[i] & 0x80) == 0 || (buf[i] & 0x40))
374 					return -1;
375 
376 				c = (c << 6) + (buf[i] & 0x3f);
377 			}
378 
379 			if (ubuf)
380 				ubuf[(*ulen)++] = c;
381 			gotone = 1;
382 		}
383 	}
384 done:
385 	return ctrl ? 0 : (gotone ? 2 : 1);
386 }
387 
388 /*
389  * Decide whether some text looks like UTF-8 with BOM. If there is no
390  * BOM, return -1; otherwise return the result of looks_utf8 on the
391  * rest of the text.
392  */
393 private int
looks_utf8_with_BOM(const unsigned char * buf,size_t nbytes,unicodechar * ubuf,size_t * ulen)394 looks_utf8_with_BOM(const unsigned char *buf, size_t nbytes, unicodechar *ubuf,
395     size_t *ulen)
396 {
397 	if (nbytes > 3 && buf[0] == 0xef && buf[1] == 0xbb && buf[2] == 0xbf)
398 		return file_looks_utf8(buf + 3, nbytes - 3, ubuf, ulen);
399 	else
400 		return -1;
401 }
402 
403 private int
looks_utf7(const unsigned char * buf,size_t nbytes,unicodechar * ubuf,size_t * ulen)404 looks_utf7(const unsigned char *buf, size_t nbytes, unicodechar *ubuf, size_t *ulen)
405 {
406 	if (nbytes > 4 && buf[0] == '+' && buf[1] == '/' && buf[2] == 'v')
407 		switch (buf[3]) {
408 		case '8':
409 		case '9':
410 		case '+':
411 		case '/':
412 			if (ubuf)
413 				*ulen = 0;
414 			return 1;
415 		default:
416 			return -1;
417 		}
418 	else
419 		return -1;
420 }
421 
422 private int
looks_ucs16(const unsigned char * bf,size_t nbytes,unicodechar * ubf,size_t * ulen)423 looks_ucs16(const unsigned char *bf, size_t nbytes, unicodechar *ubf,
424     size_t *ulen)
425 {
426 	int bigend;
427 	size_t i;
428 
429 	if (nbytes < 2)
430 		return 0;
431 
432 	if (bf[0] == 0xff && bf[1] == 0xfe)
433 		bigend = 0;
434 	else if (bf[0] == 0xfe && bf[1] == 0xff)
435 		bigend = 1;
436 	else
437 		return 0;
438 
439 	*ulen = 0;
440 
441 	for (i = 2; i + 1 < nbytes; i += 2) {
442 		/* XXX fix to properly handle chars > 65536 */
443 
444 		if (bigend)
445 			ubf[(*ulen)++] = bf[i + 1]
446 			    | (CAST(unicodechar, bf[i]) << 8);
447 		else
448 			ubf[(*ulen)++] = bf[i]
449 			    | (CAST(unicodechar, bf[i + 1]) << 8);
450 
451 		if (ubf[*ulen - 1] == 0xfffe)
452 			return 0;
453 		if (ubf[*ulen - 1] < 128 &&
454 		    text_chars[CAST(size_t, ubf[*ulen - 1])] != T)
455 			return 0;
456 	}
457 
458 	return 1 + bigend;
459 }
460 
461 private int
looks_ucs32(const unsigned char * bf,size_t nbytes,unicodechar * ubf,size_t * ulen)462 looks_ucs32(const unsigned char *bf, size_t nbytes, unicodechar *ubf,
463     size_t *ulen)
464 {
465 	int bigend;
466 	size_t i;
467 
468 	if (nbytes < 4)
469 		return 0;
470 
471 	if (bf[0] == 0xff && bf[1] == 0xfe && bf[2] == 0 && bf[3] == 0)
472 		bigend = 0;
473 	else if (bf[0] == 0 && bf[1] == 0 && bf[2] == 0xfe && bf[3] == 0xff)
474 		bigend = 1;
475 	else
476 		return 0;
477 
478 	*ulen = 0;
479 
480 	for (i = 4; i + 3 < nbytes; i += 4) {
481 		/* XXX fix to properly handle chars > 65536 */
482 
483 		if (bigend)
484 			ubf[(*ulen)++] = CAST(unicodechar, bf[i + 3])
485 			    | (CAST(unicodechar, bf[i + 2]) << 8)
486 			    | (CAST(unicodechar, bf[i + 1]) << 16)
487 			    | (CAST(unicodechar, bf[i]) << 24);
488 		else
489 			ubf[(*ulen)++] = CAST(unicodechar, bf[i + 0])
490 			    | (CAST(unicodechar, bf[i + 1]) << 8)
491 			    | (CAST(unicodechar, bf[i + 2]) << 16)
492 			    | (CAST(unicodechar, bf[i + 3]) << 24);
493 
494 		if (ubf[*ulen - 1] == 0xfffe)
495 			return 0;
496 		if (ubf[*ulen - 1] < 128 &&
497 		    text_chars[CAST(size_t, ubf[*ulen - 1])] != T)
498 			return 0;
499 	}
500 
501 	return 1 + bigend;
502 }
503 #undef F
504 #undef T
505 #undef I
506 #undef X
507 
508 /*
509  * This table maps each EBCDIC character to an (8-bit extended) ASCII
510  * character, as specified in the rationale for the dd(1) command in
511  * draft 11.2 (September, 1991) of the POSIX P1003.2 standard.
512  *
513  * Unfortunately it does not seem to correspond exactly to any of the
514  * five variants of EBCDIC documented in IBM's _Enterprise Systems
515  * Architecture/390: Principles of Operation_, SA22-7201-06, Seventh
516  * Edition, July, 1999, pp. I-1 - I-4.
517  *
518  * Fortunately, though, all versions of EBCDIC, including this one, agree
519  * on most of the printing characters that also appear in (7-bit) ASCII.
520  * Of these, only '|', '!', '~', '^', '[', and ']' are in question at all.
521  *
522  * Fortunately too, there is general agreement that codes 0x00 through
523  * 0x3F represent control characters, 0x41 a nonbreaking space, and the
524  * remainder printing characters.
525  *
526  * This is sufficient to allow us to identify EBCDIC text and to distinguish
527  * between old-style and internationalized examples of text.
528  */
529 
530 private unsigned char ebcdic_to_ascii[] = {
531   0,   1,   2,   3, 156,   9, 134, 127, 151, 141, 142,  11,  12,  13,  14,  15,
532  16,  17,  18,  19, 157, 133,   8, 135,  24,  25, 146, 143,  28,  29,  30,  31,
533 128, 129, 130, 131, 132,  10,  23,  27, 136, 137, 138, 139, 140,   5,   6,   7,
534 144, 145,  22, 147, 148, 149, 150,   4, 152, 153, 154, 155,  20,  21, 158,  26,
535 ' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|',
536 '&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~',
537 '-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?',
538 186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'','=', '"',
539 195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201,
540 202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208,
541 209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215,
542 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231,
543 '{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237,
544 '}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243,
545 '\\',159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249,
546 '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255
547 };
548 
549 #ifdef notdef
550 /*
551  * The following EBCDIC-to-ASCII table may relate more closely to reality,
552  * or at least to modern reality.  It comes from
553  *
554  *   http://ftp.s390.ibm.com/products/oe/bpxqp9.html
555  *
556  * and maps the characters of EBCDIC code page 1047 (the code used for
557  * Unix-derived software on IBM's 390 systems) to the corresponding
558  * characters from ISO 8859-1.
559  *
560  * If this table is used instead of the above one, some of the special
561  * cases for the NEL character can be taken out of the code.
562  */
563 
564 private unsigned char ebcdic_1047_to_8859[] = {
565 0x00,0x01,0x02,0x03,0x9C,0x09,0x86,0x7F,0x97,0x8D,0x8E,0x0B,0x0C,0x0D,0x0E,0x0F,
566 0x10,0x11,0x12,0x13,0x9D,0x0A,0x08,0x87,0x18,0x19,0x92,0x8F,0x1C,0x1D,0x1E,0x1F,
567 0x80,0x81,0x82,0x83,0x84,0x85,0x17,0x1B,0x88,0x89,0x8A,0x8B,0x8C,0x05,0x06,0x07,
568 0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9A,0x9B,0x14,0x15,0x9E,0x1A,
569 0x20,0xA0,0xE2,0xE4,0xE0,0xE1,0xE3,0xE5,0xE7,0xF1,0xA2,0x2E,0x3C,0x28,0x2B,0x7C,
570 0x26,0xE9,0xEA,0xEB,0xE8,0xED,0xEE,0xEF,0xEC,0xDF,0x21,0x24,0x2A,0x29,0x3B,0x5E,
571 0x2D,0x2F,0xC2,0xC4,0xC0,0xC1,0xC3,0xC5,0xC7,0xD1,0xA6,0x2C,0x25,0x5F,0x3E,0x3F,
572 0xF8,0xC9,0xCA,0xCB,0xC8,0xCD,0xCE,0xCF,0xCC,0x60,0x3A,0x23,0x40,0x27,0x3D,0x22,
573 0xD8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xAB,0xBB,0xF0,0xFD,0xFE,0xB1,
574 0xB0,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0xAA,0xBA,0xE6,0xB8,0xC6,0xA4,
575 0xB5,0x7E,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7A,0xA1,0xBF,0xD0,0x5B,0xDE,0xAE,
576 0xAC,0xA3,0xA5,0xB7,0xA9,0xA7,0xB6,0xBC,0xBD,0xBE,0xDD,0xA8,0xAF,0x5D,0xB4,0xD7,
577 0x7B,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xAD,0xF4,0xF6,0xF2,0xF3,0xF5,
578 0x7D,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50,0x51,0x52,0xB9,0xFB,0xFC,0xF9,0xFA,0xFF,
579 0x5C,0xF7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0xB2,0xD4,0xD6,0xD2,0xD3,0xD5,
580 0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xB3,0xDB,0xDC,0xD9,0xDA,0x9F
581 };
582 #endif
583 
584 /*
585  * Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII.
586  */
587 private void
from_ebcdic(const unsigned char * buf,size_t nbytes,unsigned char * out)588 from_ebcdic(const unsigned char *buf, size_t nbytes, unsigned char *out)
589 {
590 	size_t i;
591 
592 	for (i = 0; i < nbytes; i++) {
593 		out[i] = ebcdic_to_ascii[buf[i]];
594 	}
595 }
596