1 /* Coding system handler (conversion, detection, etc).
2 Copyright (C) 2001-2013 Free Software Foundation, Inc.
3 Copyright (C) 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004,
4 2005, 2006, 2007, 2008, 2009, 2010, 2011
5 National Institute of Advanced Industrial Science and Technology (AIST)
6 Registration Number H14PRO021
8 National Institute of Advanced Industrial Science and Technology (AIST)
9 Registration Number H13PRO009
11 This file is part of GNU Emacs.
13 GNU Emacs is free software: you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
18 GNU Emacs is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>. */
26 /*** TABLE OF CONTENTS ***
30 2. Emacs' internal format (emacs-utf-8) handlers
33 5. Charset-base coding systems handlers
34 6. emacs-mule (old Emacs' internal format) handlers
36 8. Shift-JIS and BIG5 handlers
38 10. C library functions
39 11. Emacs Lisp library functions
44 /*** 0. General comments ***
49 A coding system is an object for an encoding mechanism that contains
50 information about how to convert byte sequences to character
51 sequences and vice versa. When we say "decode", it means converting
52 a byte sequence of a specific coding system into a character
53 sequence that is represented by Emacs' internal coding system
54 `emacs-utf-8', and when we say "encode", it means converting a
55 character sequence of emacs-utf-8 to a byte sequence of a specific
58 In Emacs Lisp, a coding system is represented by a Lisp symbol. On
59 the C level, a coding system is represented by a vector of attributes
60 stored in the hash table Vcharset_hash_table. The conversion from
61 coding system symbol to attributes vector is done by looking up
62 Vcharset_hash_table by the symbol.
64 Coding systems are classified into the following types depending on
65 the encoding mechanism. Here's a brief description of the types.
71 o Charset-base coding system
73 A coding system defined by one or more (coded) character sets.
74 Decoding and encoding are done by a code converter defined for each
77 o Old Emacs internal format (emacs-mule)
79 The coding system adopted by old versions of Emacs (20 and 21).
81 o ISO2022-base coding system
83 The most famous coding system for multiple character sets. X's
84 Compound Text, various EUCs (Extended Unix Code), and coding systems
85 used in the Internet communication such as ISO-2022-JP are all
88 o SJIS (or Shift-JIS or MS-Kanji-Code)
90 A coding system to encode character sets: ASCII, JISX0201, and
91 JISX0208. Widely used for PC's in Japan. Details are described in
96 A coding system to encode character sets: ASCII and Big5. Widely
97 used for Chinese (mainly in Taiwan and Hong Kong). Details are
98 described in section 8. In this file, when we write "big5" (all
99 lowercase), we mean the coding system, and when we write "Big5"
100 (capitalized), we mean the character set.
104 If a user wants to decode/encode text encoded in a coding system
105 not listed above, he can supply a decoder and an encoder for it in
106 CCL (Code Conversion Language) programs. Emacs executes the CCL
107 program while decoding/encoding.
111 A coding system for text containing raw eight-bit data. Emacs
112 treats each byte of source text as a character (except for
113 end-of-line conversion).
117 Like raw text, but don't do end-of-line conversion.
122 How text end-of-line is encoded depends on operating system. For
123 instance, Unix's format is just one byte of LF (line-feed) code,
124 whereas DOS's format is two-byte sequence of `carriage-return' and
125 `line-feed' codes. MacOS's format is usually one byte of
128 Since text character encoding and end-of-line encoding are
129 independent, any coding system described above can take any format
130 of end-of-line (except for no-conversion).
134 Before using a coding system for code conversion (i.e. decoding and
135 encoding), we setup a structure of type `struct coding_system'.
136 This structure keeps various information about a specific code
137 conversion (e.g. the location of source and destination data).
144 /*** GENERAL NOTES on `detect_coding_XXX ()' functions ***
146 These functions check if a byte sequence specified as a source in
147 CODING conforms to the format of XXX, and update the members of
150 Return true if the byte sequence conforms to XXX.
152 Below is the template of these functions. */
156 detect_coding_XXX (struct coding_system
*coding
,
157 struct coding_detection_info
*detect_info
)
159 const unsigned char *src
= coding
->source
;
160 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
161 bool multibytep
= coding
->src_multibyte
;
162 ptrdiff_t consumed_chars
= 0;
168 /* Get one byte from the source. If the source is exhausted, jump
169 to no_more_source:. */
172 if (! __C_conforms_to_XXX___ (c
))
174 if (! __C_strongly_suggests_XXX__ (c
))
175 found
= CATEGORY_MASK_XXX
;
177 /* The byte sequence is invalid for XXX. */
178 detect_info
->rejected
|= CATEGORY_MASK_XXX
;
182 /* The source exhausted successfully. */
183 detect_info
->found
|= found
;
188 /*** GENERAL NOTES on `decode_coding_XXX ()' functions ***
190 These functions decode a byte sequence specified as a source by
191 CODING. The resulting multibyte text goes to a place pointed to by
192 CODING->charbuf, the length of which should not exceed
193 CODING->charbuf_size;
195 These functions set the information of original and decoded texts in
196 CODING->consumed, CODING->consumed_char, and CODING->charbuf_used.
197 They also set CODING->result to one of CODING_RESULT_XXX indicating
198 how the decoding is finished.
200 Below is the template of these functions. */
204 decode_coding_XXXX (struct coding_system
*coding
)
206 const unsigned char *src
= coding
->source
+ coding
->consumed
;
207 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
208 /* SRC_BASE remembers the start position in source in each loop.
209 The loop will be exited when there's not enough source code, or
210 when there's no room in CHARBUF for a decoded character. */
211 const unsigned char *src_base
;
212 /* A buffer to produce decoded characters. */
213 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
214 int *charbuf_end
= coding
->charbuf
+ coding
->charbuf_size
;
215 bool multibytep
= coding
->src_multibyte
;
220 if (charbuf
< charbuf_end
)
221 /* No more room to produce a decoded character. */
228 if (src_base
< src_end
229 && coding
->mode
& CODING_MODE_LAST_BLOCK
)
230 /* If the source ends by partial bytes to construct a character,
231 treat them as eight-bit raw data. */
232 while (src_base
< src_end
&& charbuf
< charbuf_end
)
233 *charbuf
++ = *src_base
++;
234 /* Remember how many bytes and characters we consumed. If the
235 source is multibyte, the bytes and chars are not identical. */
236 coding
->consumed
= coding
->consumed_char
= src_base
- coding
->source
;
237 /* Remember how many characters we produced. */
238 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
242 /*** GENERAL NOTES on `encode_coding_XXX ()' functions ***
244 These functions encode SRC_BYTES length text at SOURCE of Emacs'
245 internal multibyte format by CODING. The resulting byte sequence
246 goes to a place pointed to by DESTINATION, the length of which
247 should not exceed DST_BYTES.
249 These functions set the information of original and encoded texts in
250 the members produced, produced_char, consumed, and consumed_char of
251 the structure *CODING. They also set the member result to one of
252 CODING_RESULT_XXX indicating how the encoding finished.
254 DST_BYTES zero means that source area and destination area are
255 overlapped, which means that we can produce a encoded text until it
256 reaches at the head of not-yet-encoded source text.
258 Below is a template of these functions. */
261 encode_coding_XXX (struct coding_system
*coding
)
263 bool multibytep
= coding
->dst_multibyte
;
264 int *charbuf
= coding
->charbuf
;
265 int *charbuf_end
= charbuf
->charbuf
+ coding
->charbuf_used
;
266 unsigned char *dst
= coding
->destination
+ coding
->produced
;
267 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
268 unsigned char *adjusted_dst_end
= dst_end
- _MAX_BYTES_PRODUCED_IN_LOOP_
;
269 ptrdiff_t produced_chars
= 0;
271 for (; charbuf
< charbuf_end
&& dst
< adjusted_dst_end
; charbuf
++)
274 /* Encode C into DST, and increment DST. */
276 label_no_more_destination
:
277 /* How many chars and bytes we produced. */
278 coding
->produced_char
+= produced_chars
;
279 coding
->produced
= dst
- coding
->destination
;
284 /*** 1. Preamble ***/
291 #endif /* HAVE_WCHAR_H */
294 #include "character.h"
298 #include "composite.h"
302 #include "termhooks.h"
304 Lisp_Object Vcoding_system_hash_table
;
306 static Lisp_Object Qcoding_system
, Qeol_type
;
307 static Lisp_Object Qcoding_aliases
;
308 Lisp_Object Qunix
, Qdos
;
309 static Lisp_Object Qmac
;
310 Lisp_Object Qbuffer_file_coding_system
;
311 static Lisp_Object Qpost_read_conversion
, Qpre_write_conversion
;
312 static Lisp_Object Qdefault_char
;
313 Lisp_Object Qno_conversion
, Qundecided
;
314 Lisp_Object Qcharset
, Qutf_8
;
315 static Lisp_Object Qiso_2022
;
316 static Lisp_Object Qutf_16
, Qshift_jis
, Qbig5
;
317 static Lisp_Object Qbig
, Qlittle
;
318 static Lisp_Object Qcoding_system_history
;
319 static Lisp_Object Qvalid_codes
;
320 static Lisp_Object QCcategory
, QCmnemonic
, QCdefault_char
;
321 static Lisp_Object QCdecode_translation_table
, QCencode_translation_table
;
322 static Lisp_Object QCpost_read_conversion
, QCpre_write_conversion
;
323 static Lisp_Object QCascii_compatible_p
;
325 Lisp_Object Qcall_process
, Qcall_process_region
;
326 Lisp_Object Qstart_process
, Qopen_network_stream
;
327 static Lisp_Object Qtarget_idx
;
329 static Lisp_Object Qinsufficient_source
, Qinvalid_source
, Qinterrupted
;
331 /* If a symbol has this property, evaluate the value to define the
332 symbol as a coding system. */
333 static Lisp_Object Qcoding_system_define_form
;
335 /* Format of end-of-line decided by system. This is Qunix on
336 Unix and Mac, Qdos on DOS/Windows.
337 This has an effect only for external encoding (i.e. for output to
338 file and process), not for in-buffer or Lisp string encoding. */
339 static Lisp_Object system_eol_type
;
343 Lisp_Object Qcoding_system_p
, Qcoding_system_error
;
345 /* Coding system emacs-mule and raw-text are for converting only
346 end-of-line format. */
347 Lisp_Object Qemacs_mule
, Qraw_text
;
348 Lisp_Object Qutf_8_emacs
;
350 #if defined (WINDOWSNT) || defined (CYGWIN)
351 static Lisp_Object Qutf_16le
;
354 /* Coding-systems are handed between Emacs Lisp programs and C internal
355 routines by the following three variables. */
356 /* Coding system to be used to encode text for terminal display when
357 terminal coding system is nil. */
358 struct coding_system safe_terminal_coding
;
362 Lisp_Object Qtranslation_table
;
363 Lisp_Object Qtranslation_table_id
;
364 static Lisp_Object Qtranslation_table_for_decode
;
365 static Lisp_Object Qtranslation_table_for_encode
;
367 /* Two special coding systems. */
368 static Lisp_Object Vsjis_coding_system
;
369 static Lisp_Object Vbig5_coding_system
;
371 /* ISO2022 section */
373 #define CODING_ISO_INITIAL(coding, reg) \
374 (XINT (AREF (AREF (CODING_ID_ATTRS ((coding)->id), \
375 coding_attr_iso_initial), \
379 #define CODING_ISO_REQUEST(coding, charset_id) \
380 (((charset_id) <= (coding)->max_charset_id \
381 ? ((coding)->safe_charsets[charset_id] != 255 \
382 ? (coding)->safe_charsets[charset_id] \
387 #define CODING_ISO_FLAGS(coding) \
388 ((coding)->spec.iso_2022.flags)
389 #define CODING_ISO_DESIGNATION(coding, reg) \
390 ((coding)->spec.iso_2022.current_designation[reg])
391 #define CODING_ISO_INVOCATION(coding, plane) \
392 ((coding)->spec.iso_2022.current_invocation[plane])
393 #define CODING_ISO_SINGLE_SHIFTING(coding) \
394 ((coding)->spec.iso_2022.single_shifting)
395 #define CODING_ISO_BOL(coding) \
396 ((coding)->spec.iso_2022.bol)
397 #define CODING_ISO_INVOKED_CHARSET(coding, plane) \
398 CODING_ISO_DESIGNATION ((coding), CODING_ISO_INVOCATION ((coding), (plane)))
399 #define CODING_ISO_CMP_STATUS(coding) \
400 (&(coding)->spec.iso_2022.cmp_status)
401 #define CODING_ISO_EXTSEGMENT_LEN(coding) \
402 ((coding)->spec.iso_2022.ctext_extended_segment_len)
403 #define CODING_ISO_EMBEDDED_UTF_8(coding) \
404 ((coding)->spec.iso_2022.embedded_utf_8)
406 /* Control characters of ISO2022. */
407 /* code */ /* function */
408 #define ISO_CODE_SO 0x0E /* shift-out */
409 #define ISO_CODE_SI 0x0F /* shift-in */
410 #define ISO_CODE_SS2_7 0x19 /* single-shift-2 for 7-bit code */
411 #define ISO_CODE_ESC 0x1B /* escape */
412 #define ISO_CODE_SS2 0x8E /* single-shift-2 */
413 #define ISO_CODE_SS3 0x8F /* single-shift-3 */
414 #define ISO_CODE_CSI 0x9B /* control-sequence-introducer */
416 /* All code (1-byte) of ISO2022 is classified into one of the
418 enum iso_code_class_type
420 ISO_control_0
, /* Control codes in the range
421 0x00..0x1F and 0x7F, except for the
422 following 5 codes. */
423 ISO_shift_out
, /* ISO_CODE_SO (0x0E) */
424 ISO_shift_in
, /* ISO_CODE_SI (0x0F) */
425 ISO_single_shift_2_7
, /* ISO_CODE_SS2_7 (0x19) */
426 ISO_escape
, /* ISO_CODE_ESC (0x1B) */
427 ISO_control_1
, /* Control codes in the range
428 0x80..0x9F, except for the
429 following 3 codes. */
430 ISO_single_shift_2
, /* ISO_CODE_SS2 (0x8E) */
431 ISO_single_shift_3
, /* ISO_CODE_SS3 (0x8F) */
432 ISO_control_sequence_introducer
, /* ISO_CODE_CSI (0x9B) */
433 ISO_0x20_or_0x7F
, /* Codes of the values 0x20 or 0x7F. */
434 ISO_graphic_plane_0
, /* Graphic codes in the range 0x21..0x7E. */
435 ISO_0xA0_or_0xFF
, /* Codes of the values 0xA0 or 0xFF. */
436 ISO_graphic_plane_1
/* Graphic codes in the range 0xA1..0xFE. */
439 /** The macros CODING_ISO_FLAG_XXX defines a flag bit of the
440 `iso-flags' attribute of an iso2022 coding system. */
442 /* If set, produce long-form designation sequence (e.g. ESC $ ( A)
443 instead of the correct short-form sequence (e.g. ESC $ A). */
444 #define CODING_ISO_FLAG_LONG_FORM 0x0001
446 /* If set, reset graphic planes and registers at end-of-line to the
448 #define CODING_ISO_FLAG_RESET_AT_EOL 0x0002
450 /* If set, reset graphic planes and registers before any control
451 characters to the initial state. */
452 #define CODING_ISO_FLAG_RESET_AT_CNTL 0x0004
454 /* If set, encode by 7-bit environment. */
455 #define CODING_ISO_FLAG_SEVEN_BITS 0x0008
457 /* If set, use locking-shift function. */
458 #define CODING_ISO_FLAG_LOCKING_SHIFT 0x0010
460 /* If set, use single-shift function. Overwrite
461 CODING_ISO_FLAG_LOCKING_SHIFT. */
462 #define CODING_ISO_FLAG_SINGLE_SHIFT 0x0020
464 /* If set, use designation escape sequence. */
465 #define CODING_ISO_FLAG_DESIGNATION 0x0040
467 /* If set, produce revision number sequence. */
468 #define CODING_ISO_FLAG_REVISION 0x0080
470 /* If set, produce ISO6429's direction specifying sequence. */
471 #define CODING_ISO_FLAG_DIRECTION 0x0100
473 /* If set, assume designation states are reset at beginning of line on
475 #define CODING_ISO_FLAG_INIT_AT_BOL 0x0200
477 /* If set, designation sequence should be placed at beginning of line
479 #define CODING_ISO_FLAG_DESIGNATE_AT_BOL 0x0400
481 /* If set, do not encode unsafe characters on output. */
482 #define CODING_ISO_FLAG_SAFE 0x0800
484 /* If set, extra latin codes (128..159) are accepted as a valid code
486 #define CODING_ISO_FLAG_LATIN_EXTRA 0x1000
488 #define CODING_ISO_FLAG_COMPOSITION 0x2000
490 /* #define CODING_ISO_FLAG_EUC_TW_SHIFT 0x4000 */
492 #define CODING_ISO_FLAG_USE_ROMAN 0x8000
494 #define CODING_ISO_FLAG_USE_OLDJIS 0x10000
496 #define CODING_ISO_FLAG_FULL_SUPPORT 0x100000
498 /* A character to be produced on output if encoding of the original
499 character is prohibited by CODING_ISO_FLAG_SAFE. */
500 #define CODING_INHIBIT_CHARACTER_SUBSTITUTION '?'
503 #define CODING_UTF_8_BOM(coding) \
504 ((coding)->spec.utf_8_bom)
507 #define CODING_UTF_16_BOM(coding) \
508 ((coding)->spec.utf_16.bom)
510 #define CODING_UTF_16_ENDIAN(coding) \
511 ((coding)->spec.utf_16.endian)
513 #define CODING_UTF_16_SURROGATE(coding) \
514 ((coding)->spec.utf_16.surrogate)
518 #define CODING_CCL_DECODER(coding) \
519 AREF (CODING_ID_ATTRS ((coding)->id), coding_attr_ccl_decoder)
520 #define CODING_CCL_ENCODER(coding) \
521 AREF (CODING_ID_ATTRS ((coding)->id), coding_attr_ccl_encoder)
522 #define CODING_CCL_VALIDS(coding) \
523 (SDATA (AREF (CODING_ID_ATTRS ((coding)->id), coding_attr_ccl_valids)))
525 /* Index for each coding category in `coding_categories' */
529 coding_category_iso_7
,
530 coding_category_iso_7_tight
,
531 coding_category_iso_8_1
,
532 coding_category_iso_8_2
,
533 coding_category_iso_7_else
,
534 coding_category_iso_8_else
,
535 coding_category_utf_8_auto
,
536 coding_category_utf_8_nosig
,
537 coding_category_utf_8_sig
,
538 coding_category_utf_16_auto
,
539 coding_category_utf_16_be
,
540 coding_category_utf_16_le
,
541 coding_category_utf_16_be_nosig
,
542 coding_category_utf_16_le_nosig
,
543 coding_category_charset
,
544 coding_category_sjis
,
545 coding_category_big5
,
547 coding_category_emacs_mule
,
548 /* All above are targets of code detection. */
549 coding_category_raw_text
,
550 coding_category_undecided
,
554 /* Definitions of flag bits used in detect_coding_XXXX. */
555 #define CATEGORY_MASK_ISO_7 (1 << coding_category_iso_7)
556 #define CATEGORY_MASK_ISO_7_TIGHT (1 << coding_category_iso_7_tight)
557 #define CATEGORY_MASK_ISO_8_1 (1 << coding_category_iso_8_1)
558 #define CATEGORY_MASK_ISO_8_2 (1 << coding_category_iso_8_2)
559 #define CATEGORY_MASK_ISO_7_ELSE (1 << coding_category_iso_7_else)
560 #define CATEGORY_MASK_ISO_8_ELSE (1 << coding_category_iso_8_else)
561 #define CATEGORY_MASK_UTF_8_AUTO (1 << coding_category_utf_8_auto)
562 #define CATEGORY_MASK_UTF_8_NOSIG (1 << coding_category_utf_8_nosig)
563 #define CATEGORY_MASK_UTF_8_SIG (1 << coding_category_utf_8_sig)
564 #define CATEGORY_MASK_UTF_16_AUTO (1 << coding_category_utf_16_auto)
565 #define CATEGORY_MASK_UTF_16_BE (1 << coding_category_utf_16_be)
566 #define CATEGORY_MASK_UTF_16_LE (1 << coding_category_utf_16_le)
567 #define CATEGORY_MASK_UTF_16_BE_NOSIG (1 << coding_category_utf_16_be_nosig)
568 #define CATEGORY_MASK_UTF_16_LE_NOSIG (1 << coding_category_utf_16_le_nosig)
569 #define CATEGORY_MASK_CHARSET (1 << coding_category_charset)
570 #define CATEGORY_MASK_SJIS (1 << coding_category_sjis)
571 #define CATEGORY_MASK_BIG5 (1 << coding_category_big5)
572 #define CATEGORY_MASK_CCL (1 << coding_category_ccl)
573 #define CATEGORY_MASK_EMACS_MULE (1 << coding_category_emacs_mule)
574 #define CATEGORY_MASK_RAW_TEXT (1 << coding_category_raw_text)
576 /* This value is returned if detect_coding_mask () find nothing other
577 than ASCII characters. */
578 #define CATEGORY_MASK_ANY \
579 (CATEGORY_MASK_ISO_7 \
580 | CATEGORY_MASK_ISO_7_TIGHT \
581 | CATEGORY_MASK_ISO_8_1 \
582 | CATEGORY_MASK_ISO_8_2 \
583 | CATEGORY_MASK_ISO_7_ELSE \
584 | CATEGORY_MASK_ISO_8_ELSE \
585 | CATEGORY_MASK_UTF_8_AUTO \
586 | CATEGORY_MASK_UTF_8_NOSIG \
587 | CATEGORY_MASK_UTF_8_SIG \
588 | CATEGORY_MASK_UTF_16_AUTO \
589 | CATEGORY_MASK_UTF_16_BE \
590 | CATEGORY_MASK_UTF_16_LE \
591 | CATEGORY_MASK_UTF_16_BE_NOSIG \
592 | CATEGORY_MASK_UTF_16_LE_NOSIG \
593 | CATEGORY_MASK_CHARSET \
594 | CATEGORY_MASK_SJIS \
595 | CATEGORY_MASK_BIG5 \
596 | CATEGORY_MASK_CCL \
597 | CATEGORY_MASK_EMACS_MULE)
600 #define CATEGORY_MASK_ISO_7BIT \
601 (CATEGORY_MASK_ISO_7 | CATEGORY_MASK_ISO_7_TIGHT)
603 #define CATEGORY_MASK_ISO_8BIT \
604 (CATEGORY_MASK_ISO_8_1 | CATEGORY_MASK_ISO_8_2)
606 #define CATEGORY_MASK_ISO_ELSE \
607 (CATEGORY_MASK_ISO_7_ELSE | CATEGORY_MASK_ISO_8_ELSE)
609 #define CATEGORY_MASK_ISO_ESCAPE \
610 (CATEGORY_MASK_ISO_7 \
611 | CATEGORY_MASK_ISO_7_TIGHT \
612 | CATEGORY_MASK_ISO_7_ELSE \
613 | CATEGORY_MASK_ISO_8_ELSE)
615 #define CATEGORY_MASK_ISO \
616 ( CATEGORY_MASK_ISO_7BIT \
617 | CATEGORY_MASK_ISO_8BIT \
618 | CATEGORY_MASK_ISO_ELSE)
620 #define CATEGORY_MASK_UTF_16 \
621 (CATEGORY_MASK_UTF_16_AUTO \
622 | CATEGORY_MASK_UTF_16_BE \
623 | CATEGORY_MASK_UTF_16_LE \
624 | CATEGORY_MASK_UTF_16_BE_NOSIG \
625 | CATEGORY_MASK_UTF_16_LE_NOSIG)
627 #define CATEGORY_MASK_UTF_8 \
628 (CATEGORY_MASK_UTF_8_AUTO \
629 | CATEGORY_MASK_UTF_8_NOSIG \
630 | CATEGORY_MASK_UTF_8_SIG)
632 /* Table of coding categories (Lisp symbols). This variable is for
633 internal use only. */
634 static Lisp_Object Vcoding_category_table
;
636 /* Table of coding-categories ordered by priority. */
637 static enum coding_category coding_priorities
[coding_category_max
];
639 /* Nth element is a coding context for the coding system bound to the
640 Nth coding category. */
641 static struct coding_system coding_categories
[coding_category_max
];
643 /*** Commonly used macros and functions ***/
646 #define min(a, b) ((a) < (b) ? (a) : (b))
649 #define max(a, b) ((a) > (b) ? (a) : (b))
652 #define CODING_GET_INFO(coding, attrs, charset_list) \
654 (attrs) = CODING_ID_ATTRS ((coding)->id); \
655 (charset_list) = CODING_ATTR_CHARSET_LIST (attrs); \
659 CHECK_NATNUM_CAR (Lisp_Object x
)
661 Lisp_Object tmp
= XCAR (x
);
667 CHECK_NATNUM_CDR (Lisp_Object x
)
669 Lisp_Object tmp
= XCDR (x
);
675 /* Safely get one byte from the source text pointed by SRC which ends
676 at SRC_END, and set C to that byte. If there are not enough bytes
677 in the source, it jumps to 'no_more_source'. If MULTIBYTEP,
678 and a multibyte character is found at SRC, set C to the
679 negative value of the character code. The caller should declare
680 and set these variables appropriately in advance:
681 src, src_end, multibytep */
683 #define ONE_MORE_BYTE(c) \
685 if (src == src_end) \
687 if (src_base < src) \
688 record_conversion_result \
689 (coding, CODING_RESULT_INSUFFICIENT_SRC); \
690 goto no_more_source; \
693 if (multibytep && (c & 0x80)) \
695 if ((c & 0xFE) == 0xC0) \
696 c = ((c & 1) << 6) | *src++; \
700 c = - string_char (src, &src, NULL); \
701 record_conversion_result \
702 (coding, CODING_RESULT_INVALID_SRC); \
708 /* Safely get two bytes from the source text pointed by SRC which ends
709 at SRC_END, and set C1 and C2 to those bytes while skipping the
710 heading multibyte characters. If there are not enough bytes in the
711 source, it jumps to 'no_more_source'. If MULTIBYTEP and
712 a multibyte character is found for C2, set C2 to the negative value
713 of the character code. The caller should declare and set these
714 variables appropriately in advance:
715 src, src_end, multibytep
716 It is intended that this macro is used in detect_coding_utf_16. */
718 #define TWO_MORE_BYTES(c1, c2) \
721 if (src == src_end) \
722 goto no_more_source; \
724 if (multibytep && (c1 & 0x80)) \
726 if ((c1 & 0xFE) == 0xC0) \
727 c1 = ((c1 & 1) << 6) | *src++; \
730 src += BYTES_BY_CHAR_HEAD (c1) - 1; \
735 if (src == src_end) \
736 goto no_more_source; \
738 if (multibytep && (c2 & 0x80)) \
740 if ((c2 & 0xFE) == 0xC0) \
741 c2 = ((c2 & 1) << 6) | *src++; \
748 /* Store a byte C in the place pointed by DST and increment DST to the
749 next free point, and increment PRODUCED_CHARS. The caller should
750 assure that C is 0..127, and declare and set the variable `dst'
751 appropriately in advance.
755 #define EMIT_ONE_ASCII_BYTE(c) \
762 /* Like EMIT_ONE_ASCII_BYTE but store two bytes; C1 and C2. */
764 #define EMIT_TWO_ASCII_BYTES(c1, c2) \
766 produced_chars += 2; \
767 *dst++ = (c1), *dst++ = (c2); \
771 /* Store a byte C in the place pointed by DST and increment DST to the
772 next free point, and increment PRODUCED_CHARS. If MULTIBYTEP,
773 store in an appropriate multibyte form. The caller should
774 declare and set the variables `dst' and `multibytep' appropriately
777 #define EMIT_ONE_BYTE(c) \
784 ch = BYTE8_TO_CHAR (ch); \
785 CHAR_STRING_ADVANCE (ch, dst); \
792 /* Like EMIT_ONE_BYTE, but emit two bytes; C1 and C2. */
794 #define EMIT_TWO_BYTES(c1, c2) \
796 produced_chars += 2; \
803 ch = BYTE8_TO_CHAR (ch); \
804 CHAR_STRING_ADVANCE (ch, dst); \
807 ch = BYTE8_TO_CHAR (ch); \
808 CHAR_STRING_ADVANCE (ch, dst); \
818 #define EMIT_THREE_BYTES(c1, c2, c3) \
820 EMIT_ONE_BYTE (c1); \
821 EMIT_TWO_BYTES (c2, c3); \
825 #define EMIT_FOUR_BYTES(c1, c2, c3, c4) \
827 EMIT_TWO_BYTES (c1, c2); \
828 EMIT_TWO_BYTES (c3, c4); \
833 record_conversion_result (struct coding_system
*coding
,
834 enum coding_result_code result
)
836 coding
->result
= result
;
839 case CODING_RESULT_INSUFFICIENT_SRC
:
840 Vlast_code_conversion_error
= Qinsufficient_source
;
842 case CODING_RESULT_INVALID_SRC
:
843 Vlast_code_conversion_error
= Qinvalid_source
;
845 case CODING_RESULT_INTERRUPT
:
846 Vlast_code_conversion_error
= Qinterrupted
;
848 case CODING_RESULT_INSUFFICIENT_DST
:
849 /* Don't record this error in Vlast_code_conversion_error
850 because it happens just temporarily and is resolved when the
851 whole conversion is finished. */
853 case CODING_RESULT_SUCCESS
:
856 Vlast_code_conversion_error
= intern ("Unknown error");
860 /* These wrapper macros are used to preserve validity of pointers into
861 buffer text across calls to decode_char, encode_char, etc, which
862 could cause relocation of buffers if it loads a charset map,
863 because loading a charset map allocates large structures. */
865 #define CODING_DECODE_CHAR(coding, src, src_base, src_end, charset, code, c) \
869 charset_map_loaded = 0; \
870 c = DECODE_CHAR (charset, code); \
871 if (charset_map_loaded \
872 && (offset = coding_change_source (coding))) \
875 src_base += offset; \
880 #define CODING_ENCODE_CHAR(coding, dst, dst_end, charset, c, code) \
884 charset_map_loaded = 0; \
885 code = ENCODE_CHAR (charset, c); \
886 if (charset_map_loaded \
887 && (offset = coding_change_destination (coding))) \
894 #define CODING_CHAR_CHARSET(coding, dst, dst_end, c, charset_list, code_return, charset) \
898 charset_map_loaded = 0; \
899 charset = char_charset (c, charset_list, code_return); \
900 if (charset_map_loaded \
901 && (offset = coding_change_destination (coding))) \
908 #define CODING_CHAR_CHARSET_P(coding, dst, dst_end, c, charset, result) \
912 charset_map_loaded = 0; \
913 result = CHAR_CHARSET_P (c, charset); \
914 if (charset_map_loaded \
915 && (offset = coding_change_destination (coding))) \
923 /* If there are at least BYTES length of room at dst, allocate memory
924 for coding->destination and update dst and dst_end. We don't have
925 to take care of coding->source which will be relocated. It is
926 handled by calling coding_set_source in encode_coding. */
928 #define ASSURE_DESTINATION(bytes) \
930 if (dst + (bytes) >= dst_end) \
932 ptrdiff_t more_bytes = charbuf_end - charbuf + (bytes); \
934 dst = alloc_destination (coding, more_bytes, dst); \
935 dst_end = coding->destination + coding->dst_bytes; \
940 /* Store multibyte form of the character C in P, and advance P to the
941 end of the multibyte form. This used to be like CHAR_STRING_ADVANCE
942 without ever calling MAYBE_UNIFY_CHAR, but nowadays we don't call
943 MAYBE_UNIFY_CHAR in CHAR_STRING_ADVANCE. */
945 #define CHAR_STRING_ADVANCE_NO_UNIFY(c, p) CHAR_STRING_ADVANCE(c, p)
947 /* Return the character code of character whose multibyte form is at
948 P, and advance P to the end of the multibyte form. This used to be
949 like STRING_CHAR_ADVANCE without ever calling MAYBE_UNIFY_CHAR, but
950 nowadays STRING_CHAR_ADVANCE doesn't call MAYBE_UNIFY_CHAR. */
952 #define STRING_CHAR_ADVANCE_NO_UNIFY(p) STRING_CHAR_ADVANCE(p)
954 /* Set coding->source from coding->src_object. */
957 coding_set_source (struct coding_system
*coding
)
959 if (BUFFERP (coding
->src_object
))
961 struct buffer
*buf
= XBUFFER (coding
->src_object
);
963 if (coding
->src_pos
< 0)
964 coding
->source
= BUF_GAP_END_ADDR (buf
) + coding
->src_pos_byte
;
966 coding
->source
= BUF_BYTE_ADDRESS (buf
, coding
->src_pos_byte
);
968 else if (STRINGP (coding
->src_object
))
970 coding
->source
= SDATA (coding
->src_object
) + coding
->src_pos_byte
;
974 /* Otherwise, the source is C string and is never relocated
975 automatically. Thus we don't have to update anything. */
980 /* Set coding->source from coding->src_object, and return how many
981 bytes coding->source was changed. */
984 coding_change_source (struct coding_system
*coding
)
986 const unsigned char *orig
= coding
->source
;
987 coding_set_source (coding
);
988 return coding
->source
- orig
;
992 /* Set coding->destination from coding->dst_object. */
995 coding_set_destination (struct coding_system
*coding
)
997 if (BUFFERP (coding
->dst_object
))
999 if (BUFFERP (coding
->src_object
) && coding
->src_pos
< 0)
1001 coding
->destination
= BEG_ADDR
+ coding
->dst_pos_byte
- BEG_BYTE
;
1002 coding
->dst_bytes
= (GAP_END_ADDR
1003 - (coding
->src_bytes
- coding
->consumed
)
1004 - coding
->destination
);
1008 /* We are sure that coding->dst_pos_byte is before the gap
1010 coding
->destination
= (BUF_BEG_ADDR (XBUFFER (coding
->dst_object
))
1011 + coding
->dst_pos_byte
- BEG_BYTE
);
1012 coding
->dst_bytes
= (BUF_GAP_END_ADDR (XBUFFER (coding
->dst_object
))
1013 - coding
->destination
);
1018 /* Otherwise, the destination is C string and is never relocated
1019 automatically. Thus we don't have to update anything. */
1024 /* Set coding->destination from coding->dst_object, and return how
1025 many bytes coding->destination was changed. */
1028 coding_change_destination (struct coding_system
*coding
)
1030 const unsigned char *orig
= coding
->destination
;
1031 coding_set_destination (coding
);
1032 return coding
->destination
- orig
;
1037 coding_alloc_by_realloc (struct coding_system
*coding
, ptrdiff_t bytes
)
1039 if (STRING_BYTES_BOUND
- coding
->dst_bytes
< bytes
)
1041 coding
->destination
= xrealloc (coding
->destination
,
1042 coding
->dst_bytes
+ bytes
);
1043 coding
->dst_bytes
+= bytes
;
1047 coding_alloc_by_making_gap (struct coding_system
*coding
,
1048 ptrdiff_t gap_head_used
, ptrdiff_t bytes
)
1050 if (EQ (coding
->src_object
, coding
->dst_object
))
1052 /* The gap may contain the produced data at the head and not-yet
1053 consumed data at the tail. To preserve those data, we at
1054 first make the gap size to zero, then increase the gap
1056 ptrdiff_t add
= GAP_SIZE
;
1058 GPT
+= gap_head_used
, GPT_BYTE
+= gap_head_used
;
1059 GAP_SIZE
= 0; ZV
+= add
; Z
+= add
; ZV_BYTE
+= add
; Z_BYTE
+= add
;
1061 GAP_SIZE
+= add
; ZV
-= add
; Z
-= add
; ZV_BYTE
-= add
; Z_BYTE
-= add
;
1062 GPT
-= gap_head_used
, GPT_BYTE
-= gap_head_used
;
1065 make_gap_1 (XBUFFER (coding
->dst_object
), bytes
);
1069 static unsigned char *
1070 alloc_destination (struct coding_system
*coding
, ptrdiff_t nbytes
,
1073 ptrdiff_t offset
= dst
- coding
->destination
;
1075 if (BUFFERP (coding
->dst_object
))
1077 struct buffer
*buf
= XBUFFER (coding
->dst_object
);
1079 coding_alloc_by_making_gap (coding
, dst
- BUF_GPT_ADDR (buf
), nbytes
);
1082 coding_alloc_by_realloc (coding
, nbytes
);
1083 coding_set_destination (coding
);
1084 dst
= coding
->destination
+ offset
;
1088 /** Macros for annotations. */
1090 /* An annotation data is stored in the array coding->charbuf in this
1092 [ -LENGTH ANNOTATION_MASK NCHARS ... ]
1093 LENGTH is the number of elements in the annotation.
1094 ANNOTATION_MASK is one of CODING_ANNOTATE_XXX_MASK.
1095 NCHARS is the number of characters in the text annotated.
1097 The format of the following elements depend on ANNOTATION_MASK.
1099 In the case of CODING_ANNOTATE_COMPOSITION_MASK, these elements
1101 ... NBYTES METHOD [ COMPOSITION-COMPONENTS ... ]
1103 NBYTES is the number of bytes specified in the header part of
1104 old-style emacs-mule encoding, or 0 for the other kind of
1107 METHOD is one of enum composition_method.
1109 Optional COMPOSITION-COMPONENTS are characters and composition
1112 In the case of CODING_ANNOTATE_CHARSET_MASK, one element CHARSET-ID
1115 If ANNOTATION_MASK is 0, this annotation is just a space holder to
1116 recover from an invalid annotation, and should be skipped by
1117 produce_annotation. */
1119 /* Maximum length of the header of annotation data. */
1120 #define MAX_ANNOTATION_LENGTH 5
1122 #define ADD_ANNOTATION_DATA(buf, len, mask, nchars) \
1124 *(buf)++ = -(len); \
1125 *(buf)++ = (mask); \
1126 *(buf)++ = (nchars); \
1127 coding->annotated = 1; \
1130 #define ADD_COMPOSITION_DATA(buf, nchars, nbytes, method) \
1132 ADD_ANNOTATION_DATA (buf, 5, CODING_ANNOTATE_COMPOSITION_MASK, nchars); \
1138 #define ADD_CHARSET_DATA(buf, nchars, id) \
1140 ADD_ANNOTATION_DATA (buf, 4, CODING_ANNOTATE_CHARSET_MASK, nchars); \
1145 /* Bitmasks for coding->eol_seen. */
1147 #define EOL_SEEN_NONE 0
1148 #define EOL_SEEN_LF 1
1149 #define EOL_SEEN_CR 2
1150 #define EOL_SEEN_CRLF 4
1153 /*** 2. Emacs' internal format (emacs-utf-8) ***/
1160 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
1161 Return true if a text is encoded in UTF-8. */
1163 #define UTF_8_1_OCTET_P(c) ((c) < 0x80)
1164 #define UTF_8_EXTRA_OCTET_P(c) (((c) & 0xC0) == 0x80)
1165 #define UTF_8_2_OCTET_LEADING_P(c) (((c) & 0xE0) == 0xC0)
1166 #define UTF_8_3_OCTET_LEADING_P(c) (((c) & 0xF0) == 0xE0)
1167 #define UTF_8_4_OCTET_LEADING_P(c) (((c) & 0xF8) == 0xF0)
1168 #define UTF_8_5_OCTET_LEADING_P(c) (((c) & 0xFC) == 0xF8)
1170 #define UTF_8_BOM_1 0xEF
1171 #define UTF_8_BOM_2 0xBB
1172 #define UTF_8_BOM_3 0xBF
1174 /* Unlike the other detect_coding_XXX, this function counts number of
1175 characters and check EOL format. */
1178 detect_coding_utf_8 (struct coding_system
*coding
,
1179 struct coding_detection_info
*detect_info
)
1181 const unsigned char *src
= coding
->source
, *src_base
;
1182 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
1183 bool multibytep
= coding
->src_multibyte
;
1184 ptrdiff_t consumed_chars
= 0;
1186 int nchars
= coding
->head_ascii
;
1187 int eol_seen
= coding
->eol_seen
;
1189 detect_info
->checked
|= CATEGORY_MASK_UTF_8
;
1190 /* A coding system of this category is always ASCII compatible. */
1193 if (src
== coding
->source
/* BOM should be at the head. */
1194 && src
+ 3 < src_end
/* BOM is 3-byte long. */
1195 && src
[0] == UTF_8_BOM_1
1196 && src
[1] == UTF_8_BOM_2
1197 && src
[2] == UTF_8_BOM_3
)
1206 int c
, c1
, c2
, c3
, c4
;
1210 if (c
< 0 || UTF_8_1_OCTET_P (c
))
1215 if (src
< src_end
&& *src
== '\n')
1217 eol_seen
|= EOL_SEEN_CRLF
;
1222 eol_seen
|= EOL_SEEN_CR
;
1225 eol_seen
|= EOL_SEEN_LF
;
1229 if (c1
< 0 || ! UTF_8_EXTRA_OCTET_P (c1
))
1231 if (UTF_8_2_OCTET_LEADING_P (c
))
1237 if (c2
< 0 || ! UTF_8_EXTRA_OCTET_P (c2
))
1239 if (UTF_8_3_OCTET_LEADING_P (c
))
1245 if (c3
< 0 || ! UTF_8_EXTRA_OCTET_P (c3
))
1247 if (UTF_8_4_OCTET_LEADING_P (c
))
1253 if (c4
< 0 || ! UTF_8_EXTRA_OCTET_P (c4
))
1255 if (UTF_8_5_OCTET_LEADING_P (c
))
1262 detect_info
->rejected
|= CATEGORY_MASK_UTF_8
;
1266 if (src_base
< src
&& coding
->mode
& CODING_MODE_LAST_BLOCK
)
1268 detect_info
->rejected
|= CATEGORY_MASK_UTF_8
;
1273 /* The first character 0xFFFE doesn't necessarily mean a BOM. */
1274 detect_info
->found
|= CATEGORY_MASK_UTF_8_AUTO
| CATEGORY_MASK_UTF_8_SIG
| CATEGORY_MASK_UTF_8_NOSIG
;
1278 detect_info
->rejected
|= CATEGORY_MASK_UTF_8_SIG
;
1279 if (nchars
< src_end
- coding
->source
)
1280 /* The found characters are less than source bytes, which
1281 means that we found a valid non-ASCII characters. */
1282 detect_info
->found
|= CATEGORY_MASK_UTF_8_AUTO
| CATEGORY_MASK_UTF_8_NOSIG
;
1284 coding
->detected_utf8_chars
= nchars
;
1290 decode_coding_utf_8 (struct coding_system
*coding
)
1292 const unsigned char *src
= coding
->source
+ coding
->consumed
;
1293 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
1294 const unsigned char *src_base
;
1295 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
1296 int *charbuf_end
= coding
->charbuf
+ coding
->charbuf_size
;
1297 ptrdiff_t consumed_chars
= 0, consumed_chars_base
= 0;
1298 bool multibytep
= coding
->src_multibyte
;
1299 enum utf_bom_type bom
= CODING_UTF_8_BOM (coding
);
1301 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
1302 int byte_after_cr
= -1;
1304 if (bom
!= utf_without_bom
)
1310 if (! UTF_8_3_OCTET_LEADING_P (c1
))
1315 if (! UTF_8_EXTRA_OCTET_P (c2
))
1320 if (! UTF_8_EXTRA_OCTET_P (c3
))
1324 if ((c1
!= UTF_8_BOM_1
)
1325 || (c2
!= UTF_8_BOM_2
) || (c3
!= UTF_8_BOM_3
))
1328 CODING_UTF_8_BOM (coding
) = utf_without_bom
;
1333 CODING_UTF_8_BOM (coding
) = utf_without_bom
;
1337 int c
, c1
, c2
, c3
, c4
, c5
;
1340 consumed_chars_base
= consumed_chars
;
1342 if (charbuf
>= charbuf_end
)
1344 if (byte_after_cr
>= 0)
1349 if (byte_after_cr
>= 0)
1350 c1
= byte_after_cr
, byte_after_cr
= -1;
1357 else if (UTF_8_1_OCTET_P (c1
))
1359 if (eol_dos
&& c1
== '\r')
1360 ONE_MORE_BYTE (byte_after_cr
);
1366 if (c2
< 0 || ! UTF_8_EXTRA_OCTET_P (c2
))
1368 if (UTF_8_2_OCTET_LEADING_P (c1
))
1370 c
= ((c1
& 0x1F) << 6) | (c2
& 0x3F);
1371 /* Reject overlong sequences here and below. Encoders
1372 producing them are incorrect, they can be misleading,
1373 and they mess up read/write invariance. */
1380 if (c3
< 0 || ! UTF_8_EXTRA_OCTET_P (c3
))
1382 if (UTF_8_3_OCTET_LEADING_P (c1
))
1384 c
= (((c1
& 0xF) << 12)
1385 | ((c2
& 0x3F) << 6) | (c3
& 0x3F));
1387 || (c
>= 0xd800 && c
< 0xe000)) /* surrogates (invalid) */
1393 if (c4
< 0 || ! UTF_8_EXTRA_OCTET_P (c4
))
1395 if (UTF_8_4_OCTET_LEADING_P (c1
))
1397 c
= (((c1
& 0x7) << 18) | ((c2
& 0x3F) << 12)
1398 | ((c3
& 0x3F) << 6) | (c4
& 0x3F));
1405 if (c5
< 0 || ! UTF_8_EXTRA_OCTET_P (c5
))
1407 if (UTF_8_5_OCTET_LEADING_P (c1
))
1409 c
= (((c1
& 0x3) << 24) | ((c2
& 0x3F) << 18)
1410 | ((c3
& 0x3F) << 12) | ((c4
& 0x3F) << 6)
1412 if ((c
> MAX_CHAR
) || (c
< 0x200000))
1427 consumed_chars
= consumed_chars_base
;
1429 *charbuf
++ = ASCII_BYTE_P (c
) ? c
: BYTE8_TO_CHAR (c
);
1434 coding
->consumed_char
+= consumed_chars_base
;
1435 coding
->consumed
= src_base
- coding
->source
;
1436 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
1441 encode_coding_utf_8 (struct coding_system
*coding
)
1443 bool multibytep
= coding
->dst_multibyte
;
1444 int *charbuf
= coding
->charbuf
;
1445 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
1446 unsigned char *dst
= coding
->destination
+ coding
->produced
;
1447 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
1448 ptrdiff_t produced_chars
= 0;
1451 if (CODING_UTF_8_BOM (coding
) == utf_with_bom
)
1453 ASSURE_DESTINATION (3);
1454 EMIT_THREE_BYTES (UTF_8_BOM_1
, UTF_8_BOM_2
, UTF_8_BOM_3
);
1455 CODING_UTF_8_BOM (coding
) = utf_without_bom
;
1460 int safe_room
= MAX_MULTIBYTE_LENGTH
* 2;
1462 while (charbuf
< charbuf_end
)
1464 unsigned char str
[MAX_MULTIBYTE_LENGTH
], *p
, *pend
= str
;
1466 ASSURE_DESTINATION (safe_room
);
1468 if (CHAR_BYTE8_P (c
))
1470 c
= CHAR_TO_BYTE8 (c
);
1475 CHAR_STRING_ADVANCE_NO_UNIFY (c
, pend
);
1476 for (p
= str
; p
< pend
; p
++)
1483 int safe_room
= MAX_MULTIBYTE_LENGTH
;
1485 while (charbuf
< charbuf_end
)
1487 ASSURE_DESTINATION (safe_room
);
1489 if (CHAR_BYTE8_P (c
))
1490 *dst
++ = CHAR_TO_BYTE8 (c
);
1492 CHAR_STRING_ADVANCE_NO_UNIFY (c
, dst
);
1496 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
1497 coding
->produced_char
+= produced_chars
;
1498 coding
->produced
= dst
- coding
->destination
;
1503 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
1504 Return true if a text is encoded in one of UTF-16 based coding systems. */
1506 #define UTF_16_HIGH_SURROGATE_P(val) \
1507 (((val) & 0xFC00) == 0xD800)
1509 #define UTF_16_LOW_SURROGATE_P(val) \
1510 (((val) & 0xFC00) == 0xDC00)
1514 detect_coding_utf_16 (struct coding_system
*coding
,
1515 struct coding_detection_info
*detect_info
)
1517 const unsigned char *src
= coding
->source
;
1518 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
1519 bool multibytep
= coding
->src_multibyte
;
1522 detect_info
->checked
|= CATEGORY_MASK_UTF_16
;
1523 if (coding
->mode
& CODING_MODE_LAST_BLOCK
1524 && (coding
->src_chars
& 1))
1526 detect_info
->rejected
|= CATEGORY_MASK_UTF_16
;
1530 TWO_MORE_BYTES (c1
, c2
);
1531 if ((c1
== 0xFF) && (c2
== 0xFE))
1533 detect_info
->found
|= (CATEGORY_MASK_UTF_16_LE
1534 | CATEGORY_MASK_UTF_16_AUTO
);
1535 detect_info
->rejected
|= (CATEGORY_MASK_UTF_16_BE
1536 | CATEGORY_MASK_UTF_16_BE_NOSIG
1537 | CATEGORY_MASK_UTF_16_LE_NOSIG
);
1539 else if ((c1
== 0xFE) && (c2
== 0xFF))
1541 detect_info
->found
|= (CATEGORY_MASK_UTF_16_BE
1542 | CATEGORY_MASK_UTF_16_AUTO
);
1543 detect_info
->rejected
|= (CATEGORY_MASK_UTF_16_LE
1544 | CATEGORY_MASK_UTF_16_BE_NOSIG
1545 | CATEGORY_MASK_UTF_16_LE_NOSIG
);
1549 detect_info
->rejected
|= CATEGORY_MASK_UTF_16
;
1554 /* We check the dispersion of Eth and Oth bytes where E is even and
1555 O is odd. If both are high, we assume binary data.*/
1556 unsigned char e
[256], o
[256];
1557 unsigned e_num
= 1, o_num
= 1;
1564 detect_info
->rejected
|= (CATEGORY_MASK_UTF_16_AUTO
1565 |CATEGORY_MASK_UTF_16_BE
1566 | CATEGORY_MASK_UTF_16_LE
);
1568 while ((detect_info
->rejected
& CATEGORY_MASK_UTF_16
)
1569 != CATEGORY_MASK_UTF_16
)
1571 TWO_MORE_BYTES (c1
, c2
);
1579 detect_info
->rejected
|= CATEGORY_MASK_UTF_16_BE_NOSIG
;
1586 detect_info
->rejected
|= CATEGORY_MASK_UTF_16_LE_NOSIG
;
1597 decode_coding_utf_16 (struct coding_system
*coding
)
1599 const unsigned char *src
= coding
->source
+ coding
->consumed
;
1600 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
1601 const unsigned char *src_base
;
1602 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
1603 /* We may produces at most 3 chars in one loop. */
1604 int *charbuf_end
= coding
->charbuf
+ coding
->charbuf_size
- 2;
1605 ptrdiff_t consumed_chars
= 0, consumed_chars_base
= 0;
1606 bool multibytep
= coding
->src_multibyte
;
1607 enum utf_bom_type bom
= CODING_UTF_16_BOM (coding
);
1608 enum utf_16_endian_type endian
= CODING_UTF_16_ENDIAN (coding
);
1609 int surrogate
= CODING_UTF_16_SURROGATE (coding
);
1611 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
1612 int byte_after_cr1
= -1, byte_after_cr2
= -1;
1614 if (bom
== utf_with_bom
)
1623 if (endian
== utf_16_big_endian
1624 ? c
!= 0xFEFF : c
!= 0xFFFE)
1626 /* The first two bytes are not BOM. Treat them as bytes
1627 for a normal character. */
1631 CODING_UTF_16_BOM (coding
) = utf_without_bom
;
1633 else if (bom
== utf_detect_bom
)
1635 /* We have already tried to detect BOM and failed in
1637 CODING_UTF_16_BOM (coding
) = utf_without_bom
;
1645 consumed_chars_base
= consumed_chars
;
1647 if (charbuf
>= charbuf_end
)
1649 if (byte_after_cr1
>= 0)
1654 if (byte_after_cr1
>= 0)
1655 c1
= byte_after_cr1
, byte_after_cr1
= -1;
1663 if (byte_after_cr2
>= 0)
1664 c2
= byte_after_cr2
, byte_after_cr2
= -1;
1669 *charbuf
++ = ASCII_BYTE_P (c1
) ? c1
: BYTE8_TO_CHAR (c1
);
1673 c
= (endian
== utf_16_big_endian
1674 ? ((c1
<< 8) | c2
) : ((c2
<< 8) | c1
));
1678 if (! UTF_16_LOW_SURROGATE_P (c
))
1680 if (endian
== utf_16_big_endian
)
1681 c1
= surrogate
>> 8, c2
= surrogate
& 0xFF;
1683 c1
= surrogate
& 0xFF, c2
= surrogate
>> 8;
1687 if (UTF_16_HIGH_SURROGATE_P (c
))
1688 CODING_UTF_16_SURROGATE (coding
) = surrogate
= c
;
1694 c
= ((surrogate
- 0xD800) << 10) | (c
- 0xDC00);
1695 CODING_UTF_16_SURROGATE (coding
) = surrogate
= 0;
1696 *charbuf
++ = 0x10000 + c
;
1701 if (UTF_16_HIGH_SURROGATE_P (c
))
1702 CODING_UTF_16_SURROGATE (coding
) = surrogate
= c
;
1705 if (eol_dos
&& c
== '\r')
1707 ONE_MORE_BYTE (byte_after_cr1
);
1708 ONE_MORE_BYTE (byte_after_cr2
);
1716 coding
->consumed_char
+= consumed_chars_base
;
1717 coding
->consumed
= src_base
- coding
->source
;
1718 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
1722 encode_coding_utf_16 (struct coding_system
*coding
)
1724 bool multibytep
= coding
->dst_multibyte
;
1725 int *charbuf
= coding
->charbuf
;
1726 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
1727 unsigned char *dst
= coding
->destination
+ coding
->produced
;
1728 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
1730 enum utf_bom_type bom
= CODING_UTF_16_BOM (coding
);
1731 bool big_endian
= CODING_UTF_16_ENDIAN (coding
) == utf_16_big_endian
;
1732 ptrdiff_t produced_chars
= 0;
1735 if (bom
!= utf_without_bom
)
1737 ASSURE_DESTINATION (safe_room
);
1739 EMIT_TWO_BYTES (0xFE, 0xFF);
1741 EMIT_TWO_BYTES (0xFF, 0xFE);
1742 CODING_UTF_16_BOM (coding
) = utf_without_bom
;
1745 while (charbuf
< charbuf_end
)
1747 ASSURE_DESTINATION (safe_room
);
1749 if (c
> MAX_UNICODE_CHAR
)
1750 c
= coding
->default_char
;
1755 EMIT_TWO_BYTES (c
>> 8, c
& 0xFF);
1757 EMIT_TWO_BYTES (c
& 0xFF, c
>> 8);
1764 c1
= (c
>> 10) + 0xD800;
1765 c2
= (c
& 0x3FF) + 0xDC00;
1767 EMIT_FOUR_BYTES (c1
>> 8, c1
& 0xFF, c2
>> 8, c2
& 0xFF);
1769 EMIT_FOUR_BYTES (c1
& 0xFF, c1
>> 8, c2
& 0xFF, c2
>> 8);
1772 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
1773 coding
->produced
= dst
- coding
->destination
;
1774 coding
->produced_char
+= produced_chars
;
1779 /*** 6. Old Emacs' internal format (emacs-mule) ***/
1781 /* Emacs' internal format for representation of multiple character
1782 sets is a kind of multi-byte encoding, i.e. characters are
1783 represented by variable-length sequences of one-byte codes.
1785 ASCII characters and control characters (e.g. `tab', `newline') are
1786 represented by one-byte sequences which are their ASCII codes, in
1787 the range 0x00 through 0x7F.
1789 8-bit characters of the range 0x80..0x9F are represented by
1790 two-byte sequences of LEADING_CODE_8_BIT_CONTROL and (their 8-bit
1793 8-bit characters of the range 0xA0..0xFF are represented by
1794 one-byte sequences which are their 8-bit code.
1796 The other characters are represented by a sequence of `base
1797 leading-code', optional `extended leading-code', and one or two
1798 `position-code's. The length of the sequence is determined by the
1799 base leading-code. Leading-code takes the range 0x81 through 0x9D,
1800 whereas extended leading-code and position-code take the range 0xA0
1801 through 0xFF. See `charset.h' for more details about leading-code
1804 --- CODE RANGE of Emacs' internal format ---
1808 eight-bit-control LEADING_CODE_8_BIT_CONTROL + 0xA0..0xBF
1809 eight-bit-graphic 0xA0..0xBF
1810 ELSE 0x81..0x9D + [0xA0..0xFF]+
1811 ---------------------------------------------
1813 As this is the internal character representation, the format is
1814 usually not used externally (i.e. in a file or in a data sent to a
1815 process). But, it is possible to have a text externally in this
1816 format (i.e. by encoding by the coding system `emacs-mule').
1818 In that case, a sequence of one-byte codes has a slightly different
1821 At first, all characters in eight-bit-control are represented by
1822 one-byte sequences which are their 8-bit code.
1824 Next, character composition data are represented by the byte
1825 sequence of the form: 0x80 METHOD BYTES CHARS COMPONENT ...,
1827 METHOD is 0xF2 plus one of composition method (enum
1828 composition_method),
1830 BYTES is 0xA0 plus a byte length of this composition data,
1832 CHARS is 0xA0 plus a number of characters composed by this
1835 COMPONENTs are characters of multibyte form or composition
1836 rules encoded by two-byte of ASCII codes.
1838 In addition, for backward compatibility, the following formats are
1839 also recognized as composition data on decoding.
1842 0x80 0xFF MSEQ RULE MSEQ RULE ... MSEQ
1845 MSEQ is a multibyte form but in these special format:
1846 ASCII: 0xA0 ASCII_CODE+0x80,
1847 other: LEADING_CODE+0x20 FOLLOWING-BYTE ...,
1848 RULE is a one byte code of the range 0xA0..0xF0 that
1849 represents a composition rule.
1852 char emacs_mule_bytes
[256];
1855 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
1856 Return true if a text is encoded in 'emacs-mule'. */
1859 detect_coding_emacs_mule (struct coding_system
*coding
,
1860 struct coding_detection_info
*detect_info
)
1862 const unsigned char *src
= coding
->source
, *src_base
;
1863 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
1864 bool multibytep
= coding
->src_multibyte
;
1865 ptrdiff_t consumed_chars
= 0;
1869 detect_info
->checked
|= CATEGORY_MASK_EMACS_MULE
;
1870 /* A coding system of this category is always ASCII compatible. */
1871 src
+= coding
->head_ascii
;
1881 /* Perhaps the start of composite character. We simply skip
1882 it because analyzing it is too heavy for detecting. But,
1883 at least, we check that the composite character
1884 constitutes of more than 4 bytes. */
1885 const unsigned char *src_start
;
1895 if (src
- src_start
<= 4)
1897 found
= CATEGORY_MASK_EMACS_MULE
;
1905 && (c
== ISO_CODE_ESC
|| c
== ISO_CODE_SI
|| c
== ISO_CODE_SO
))
1910 int more_bytes
= emacs_mule_bytes
[c
] - 1;
1912 while (more_bytes
> 0)
1917 src
--; /* Unread the last byte. */
1922 if (more_bytes
!= 0)
1924 found
= CATEGORY_MASK_EMACS_MULE
;
1927 detect_info
->rejected
|= CATEGORY_MASK_EMACS_MULE
;
1931 if (src_base
< src
&& coding
->mode
& CODING_MODE_LAST_BLOCK
)
1933 detect_info
->rejected
|= CATEGORY_MASK_EMACS_MULE
;
1936 detect_info
->found
|= found
;
1941 /* Parse emacs-mule multibyte sequence at SRC and return the decoded
1942 character. If CMP_STATUS indicates that we must expect MSEQ or
1943 RULE described above, decode it and return the negative value of
1944 the decoded character or rule. If an invalid byte is found, return
1945 -1. If SRC is too short, return -2. */
1948 emacs_mule_char (struct coding_system
*coding
, const unsigned char *src
,
1949 int *nbytes
, int *nchars
, int *id
,
1950 struct composition_status
*cmp_status
)
1952 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
1953 const unsigned char *src_base
= src
;
1954 bool multibytep
= coding
->src_multibyte
;
1958 int consumed_chars
= 0;
1959 bool mseq_found
= 0;
1965 charset_ID
= emacs_mule_charset
[0];
1971 if (cmp_status
->state
!= COMPOSING_NO
1972 && cmp_status
->old_form
)
1974 if (cmp_status
->state
== COMPOSING_CHAR
)
1989 *nbytes
= src
- src_base
;
1990 *nchars
= consumed_chars
;
1998 switch (emacs_mule_bytes
[c
])
2001 if ((charset_ID
= emacs_mule_charset
[c
]) < 0)
2010 if (c
== EMACS_MULE_LEADING_CODE_PRIVATE_11
2011 || c
== EMACS_MULE_LEADING_CODE_PRIVATE_12
)
2014 if (c
< 0xA0 || (charset_ID
= emacs_mule_charset
[c
]) < 0)
2023 if ((charset_ID
= emacs_mule_charset
[c
]) < 0)
2028 code
= (c
& 0x7F) << 8;
2038 if (c
< 0 || (charset_ID
= emacs_mule_charset
[c
]) < 0)
2043 code
= (c
& 0x7F) << 8;
2052 charset_ID
= ASCII_BYTE_P (code
) ? charset_ascii
: charset_eight_bit
;
2058 CODING_DECODE_CHAR (coding
, src
, src_base
, src_end
,
2059 CHARSET_FROM_ID (charset_ID
), code
, c
);
2063 *nbytes
= src
- src_base
;
2064 *nchars
= consumed_chars
;
2067 return (mseq_found
? -c
: c
);
2077 /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions". */
2079 /* Handle these composition sequence ('|': the end of header elements,
2080 BYTES and CHARS >= 0xA0):
2082 (1) relative composition: 0x80 0xF2 BYTES CHARS | CHAR ...
2083 (2) altchar composition: 0x80 0xF4 BYTES CHARS | ALT ... ALT CHAR ...
2084 (3) alt&rule composition: 0x80 0xF5 BYTES CHARS | ALT RULE ... ALT CHAR ...
2088 (4) relative composition: 0x80 | MSEQ ... MSEQ
2089 (5) rulebase composition: 0x80 0xFF | MSEQ MRULE ... MSEQ
2091 When the starter 0x80 and the following header elements are found,
2092 this annotation header is produced.
2094 [ -LENGTH(==-5) CODING_ANNOTATE_COMPOSITION_MASK NCHARS NBYTES METHOD ]
2096 NCHARS is CHARS - 0xA0 for (1), (2), (3), and 0 for (4), (5).
2097 NBYTES is BYTES - 0xA0 for (1), (2), (3), and 0 for (4), (5).
2099 Then, upon reading the following elements, these codes are produced
2100 until the composition end is found:
2103 (2) ALT ... ALT CHAR ... CHAR
2104 (3) ALT -2 DECODED-RULE ALT -2 DECODED-RULE ... ALT CHAR ... CHAR
2106 (5) CHAR -2 DECODED-RULE CHAR -2 DECODED-RULE ... CHAR
2108 When the composition end is found, LENGTH and NCHARS in the
2109 annotation header is updated as below:
2111 (1) LENGTH: unchanged, NCHARS: unchanged
2112 (2) LENGTH: length of the whole sequence minus NCHARS, NCHARS: unchanged
2113 (3) LENGTH: length of the whole sequence minus NCHARS, NCHARS: unchanged
2114 (4) LENGTH: unchanged, NCHARS: number of CHARs
2115 (5) LENGTH: unchanged, NCHARS: number of CHARs
2117 If an error is found while composing, the annotation header is
2118 changed to the original composition header (plus filler -1s) as
2121 (1),(2),(3) [ 0x80 0xF2+METHOD BYTES CHARS -1 ]
2122 (5) [ 0x80 0xFF -1 -1- -1 ]
2124 and the sequence [ -2 DECODED-RULE ] is changed to the original
2125 byte sequence as below:
2126 o the original byte sequence is B: [ B -1 ]
2127 o the original byte sequence is B1 B2: [ B1 B2 ]
2129 Most of the routines are implemented by macros because many
2130 variables and labels in the caller decode_coding_emacs_mule must be
2131 accessible, and they are usually called just once (thus doesn't
2132 increase the size of compiled object). */
2134 /* Decode a composition rule represented by C as a component of
2135 composition sequence of Emacs 20 style. Set RULE to the decoded
2138 #define DECODE_EMACS_MULE_COMPOSITION_RULE_20(c, rule) \
2143 if (c < 0 || c >= 81) \
2144 goto invalid_code; \
2145 gref = c / 9, nref = c % 9; \
2146 if (gref == 4) gref = 10; \
2147 if (nref == 4) nref = 10; \
2148 rule = COMPOSITION_ENCODE_RULE (gref, nref); \
2152 /* Decode a composition rule represented by C and the following byte
2153 at SRC as a component of composition sequence of Emacs 21 style.
2154 Set RULE to the decoded rule. */
2156 #define DECODE_EMACS_MULE_COMPOSITION_RULE_21(c, rule) \
2161 if (gref < 0 || gref >= 81) \
2162 goto invalid_code; \
2163 ONE_MORE_BYTE (c); \
2165 if (nref < 0 || nref >= 81) \
2166 goto invalid_code; \
2167 rule = COMPOSITION_ENCODE_RULE (gref, nref); \
2171 /* Start of Emacs 21 style format. The first three bytes at SRC are
2172 (METHOD - 0xF2), (BYTES - 0xA0), (CHARS - 0xA0), where BYTES is the
2173 byte length of this composition information, CHARS is the number of
2174 characters composed by this composition. */
2176 #define DECODE_EMACS_MULE_21_COMPOSITION() \
2178 enum composition_method method = c - 0xF2; \
2179 int nbytes, nchars; \
2181 ONE_MORE_BYTE (c); \
2183 goto invalid_code; \
2184 nbytes = c - 0xA0; \
2185 if (nbytes < 3 || (method == COMPOSITION_RELATIVE && nbytes != 4)) \
2186 goto invalid_code; \
2187 ONE_MORE_BYTE (c); \
2188 nchars = c - 0xA0; \
2189 if (nchars <= 0 || nchars >= MAX_COMPOSITION_COMPONENTS) \
2190 goto invalid_code; \
2191 cmp_status->old_form = 0; \
2192 cmp_status->method = method; \
2193 if (method == COMPOSITION_RELATIVE) \
2194 cmp_status->state = COMPOSING_CHAR; \
2196 cmp_status->state = COMPOSING_COMPONENT_CHAR; \
2197 cmp_status->length = MAX_ANNOTATION_LENGTH; \
2198 cmp_status->nchars = nchars; \
2199 cmp_status->ncomps = nbytes - 4; \
2200 ADD_COMPOSITION_DATA (charbuf, nchars, nbytes, method); \
2204 /* Start of Emacs 20 style format for relative composition. */
2206 #define DECODE_EMACS_MULE_20_RELATIVE_COMPOSITION() \
2208 cmp_status->old_form = 1; \
2209 cmp_status->method = COMPOSITION_RELATIVE; \
2210 cmp_status->state = COMPOSING_CHAR; \
2211 cmp_status->length = MAX_ANNOTATION_LENGTH; \
2212 cmp_status->nchars = cmp_status->ncomps = 0; \
2213 ADD_COMPOSITION_DATA (charbuf, 0, 0, cmp_status->method); \
2217 /* Start of Emacs 20 style format for rule-base composition. */
2219 #define DECODE_EMACS_MULE_20_RULEBASE_COMPOSITION() \
2221 cmp_status->old_form = 1; \
2222 cmp_status->method = COMPOSITION_WITH_RULE; \
2223 cmp_status->state = COMPOSING_CHAR; \
2224 cmp_status->length = MAX_ANNOTATION_LENGTH; \
2225 cmp_status->nchars = cmp_status->ncomps = 0; \
2226 ADD_COMPOSITION_DATA (charbuf, 0, 0, cmp_status->method); \
2230 #define DECODE_EMACS_MULE_COMPOSITION_START() \
2232 const unsigned char *current_src = src; \
2234 ONE_MORE_BYTE (c); \
2236 goto invalid_code; \
2237 if (c - 0xF2 >= COMPOSITION_RELATIVE \
2238 && c - 0xF2 <= COMPOSITION_WITH_RULE_ALTCHARS) \
2239 DECODE_EMACS_MULE_21_COMPOSITION (); \
2240 else if (c < 0xA0) \
2241 goto invalid_code; \
2242 else if (c < 0xC0) \
2244 DECODE_EMACS_MULE_20_RELATIVE_COMPOSITION (); \
2245 /* Re-read C as a composition component. */ \
2246 src = current_src; \
2248 else if (c == 0xFF) \
2249 DECODE_EMACS_MULE_20_RULEBASE_COMPOSITION (); \
2251 goto invalid_code; \
2254 #define EMACS_MULE_COMPOSITION_END() \
2256 int idx = - cmp_status->length; \
2258 if (cmp_status->old_form) \
2259 charbuf[idx + 2] = cmp_status->nchars; \
2260 else if (cmp_status->method > COMPOSITION_RELATIVE) \
2261 charbuf[idx] = charbuf[idx + 2] - cmp_status->length; \
2262 cmp_status->state = COMPOSING_NO; \
2267 emacs_mule_finish_composition (int *charbuf
,
2268 struct composition_status
*cmp_status
)
2270 int idx
= - cmp_status
->length
;
2273 if (cmp_status
->old_form
&& cmp_status
->nchars
> 0)
2275 charbuf
[idx
+ 2] = cmp_status
->nchars
;
2277 if (cmp_status
->method
== COMPOSITION_WITH_RULE
2278 && cmp_status
->state
== COMPOSING_CHAR
)
2280 /* The last rule was invalid. */
2281 int rule
= charbuf
[-1] + 0xA0;
2283 charbuf
[-2] = BYTE8_TO_CHAR (rule
);
2290 charbuf
[idx
++] = BYTE8_TO_CHAR (0x80);
2292 if (cmp_status
->method
== COMPOSITION_WITH_RULE
)
2294 charbuf
[idx
++] = BYTE8_TO_CHAR (0xFF);
2295 charbuf
[idx
++] = -3;
2301 int nchars
= charbuf
[idx
+ 1] + 0xA0;
2302 int nbytes
= charbuf
[idx
+ 2] + 0xA0;
2304 charbuf
[idx
++] = BYTE8_TO_CHAR (0xF2 + cmp_status
->method
);
2305 charbuf
[idx
++] = BYTE8_TO_CHAR (nbytes
);
2306 charbuf
[idx
++] = BYTE8_TO_CHAR (nchars
);
2307 charbuf
[idx
++] = -1;
2311 cmp_status
->state
= COMPOSING_NO
;
2315 #define EMACS_MULE_MAYBE_FINISH_COMPOSITION() \
2317 if (cmp_status->state != COMPOSING_NO) \
2318 char_offset += emacs_mule_finish_composition (charbuf, cmp_status); \
2323 decode_coding_emacs_mule (struct coding_system
*coding
)
2325 const unsigned char *src
= coding
->source
+ coding
->consumed
;
2326 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
2327 const unsigned char *src_base
;
2328 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
2329 /* We may produce two annotations (charset and composition) in one
2330 loop and one more charset annotation at the end. */
2332 = coding
->charbuf
+ coding
->charbuf_size
- (MAX_ANNOTATION_LENGTH
* 3)
2333 /* We can produce up to 2 characters in a loop. */
2335 ptrdiff_t consumed_chars
= 0, consumed_chars_base
;
2336 bool multibytep
= coding
->src_multibyte
;
2337 ptrdiff_t char_offset
= coding
->produced_char
;
2338 ptrdiff_t last_offset
= char_offset
;
2339 int last_id
= charset_ascii
;
2341 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
2342 int byte_after_cr
= -1;
2343 struct composition_status
*cmp_status
= &coding
->spec
.emacs_mule
.cmp_status
;
2345 if (cmp_status
->state
!= COMPOSING_NO
)
2349 if (charbuf_end
- charbuf
< cmp_status
->length
)
2351 for (i
= 0; i
< cmp_status
->length
; i
++)
2352 *charbuf
++ = cmp_status
->carryover
[i
];
2353 coding
->annotated
= 1;
2358 int c
, id
IF_LINT (= 0);
2361 consumed_chars_base
= consumed_chars
;
2363 if (charbuf
>= charbuf_end
)
2365 if (byte_after_cr
>= 0)
2370 if (byte_after_cr
>= 0)
2371 c
= byte_after_cr
, byte_after_cr
= -1;
2375 if (c
< 0 || c
== 0x80)
2377 EMACS_MULE_MAYBE_FINISH_COMPOSITION ();
2384 DECODE_EMACS_MULE_COMPOSITION_START ();
2390 if (eol_dos
&& c
== '\r')
2391 ONE_MORE_BYTE (byte_after_cr
);
2393 if (cmp_status
->state
!= COMPOSING_NO
)
2395 if (cmp_status
->old_form
)
2396 EMACS_MULE_MAYBE_FINISH_COMPOSITION ();
2397 else if (cmp_status
->state
>= COMPOSING_COMPONENT_CHAR
)
2398 cmp_status
->ncomps
--;
2403 int nchars
IF_LINT (= 0), nbytes
IF_LINT (= 0);
2404 /* emacs_mule_char can load a charset map from a file, which
2405 allocates a large structure and might cause buffer text
2406 to be relocated as result. Thus, we need to remember the
2407 original pointer to buffer text, and fix up all related
2408 pointers after the call. */
2409 const unsigned char *orig
= coding
->source
;
2412 c
= emacs_mule_char (coding
, src_base
, &nbytes
, &nchars
, &id
,
2414 offset
= coding
->source
- orig
;
2428 src
= src_base
+ nbytes
;
2429 consumed_chars
= consumed_chars_base
+ nchars
;
2430 if (cmp_status
->state
>= COMPOSING_COMPONENT_CHAR
)
2431 cmp_status
->ncomps
-= nchars
;
2434 /* Now if C >= 0, we found a normally encoded character, if C <
2435 0, we found an old-style composition component character or
2438 if (cmp_status
->state
== COMPOSING_NO
)
2442 if (last_id
!= charset_ascii
)
2443 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
,
2446 last_offset
= char_offset
;
2451 else if (cmp_status
->state
== COMPOSING_CHAR
)
2453 if (cmp_status
->old_form
)
2457 EMACS_MULE_MAYBE_FINISH_COMPOSITION ();
2464 cmp_status
->nchars
++;
2465 cmp_status
->length
++;
2466 if (cmp_status
->nchars
== MAX_COMPOSITION_COMPONENTS
)
2467 EMACS_MULE_COMPOSITION_END ();
2468 else if (cmp_status
->method
== COMPOSITION_WITH_RULE
)
2469 cmp_status
->state
= COMPOSING_RULE
;
2475 cmp_status
->length
++;
2476 cmp_status
->nchars
--;
2477 if (cmp_status
->nchars
== 0)
2478 EMACS_MULE_COMPOSITION_END ();
2481 else if (cmp_status
->state
== COMPOSING_RULE
)
2487 EMACS_MULE_COMPOSITION_END ();
2494 DECODE_EMACS_MULE_COMPOSITION_RULE_20 (c
, rule
);
2499 cmp_status
->length
+= 2;
2500 cmp_status
->state
= COMPOSING_CHAR
;
2503 else if (cmp_status
->state
== COMPOSING_COMPONENT_CHAR
)
2506 cmp_status
->length
++;
2507 if (cmp_status
->ncomps
== 0)
2508 cmp_status
->state
= COMPOSING_CHAR
;
2509 else if (cmp_status
->ncomps
> 0)
2511 if (cmp_status
->method
== COMPOSITION_WITH_RULE_ALTCHARS
)
2512 cmp_status
->state
= COMPOSING_COMPONENT_RULE
;
2515 EMACS_MULE_MAYBE_FINISH_COMPOSITION ();
2517 else /* COMPOSING_COMPONENT_RULE */
2521 DECODE_EMACS_MULE_COMPOSITION_RULE_21 (c
, rule
);
2526 cmp_status
->length
+= 2;
2527 cmp_status
->ncomps
--;
2528 if (cmp_status
->ncomps
> 0)
2529 cmp_status
->state
= COMPOSING_COMPONENT_CHAR
;
2531 EMACS_MULE_MAYBE_FINISH_COMPOSITION ();
2536 EMACS_MULE_MAYBE_FINISH_COMPOSITION ();
2538 consumed_chars
= consumed_chars_base
;
2540 *charbuf
++ = ASCII_BYTE_P (c
) ? c
: BYTE8_TO_CHAR (c
);
2546 if (cmp_status
->state
!= COMPOSING_NO
)
2548 if (coding
->mode
& CODING_MODE_LAST_BLOCK
)
2549 EMACS_MULE_MAYBE_FINISH_COMPOSITION ();
2554 charbuf
-= cmp_status
->length
;
2555 for (i
= 0; i
< cmp_status
->length
; i
++)
2556 cmp_status
->carryover
[i
] = charbuf
[i
];
2559 if (last_id
!= charset_ascii
)
2560 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
2561 coding
->consumed_char
+= consumed_chars_base
;
2562 coding
->consumed
= src_base
- coding
->source
;
2563 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
2567 #define EMACS_MULE_LEADING_CODES(id, codes) \
2570 codes[0] = id, codes[1] = 0; \
2571 else if (id < 0xE0) \
2572 codes[0] = 0x9A, codes[1] = id; \
2573 else if (id < 0xF0) \
2574 codes[0] = 0x9B, codes[1] = id; \
2575 else if (id < 0xF5) \
2576 codes[0] = 0x9C, codes[1] = id; \
2578 codes[0] = 0x9D, codes[1] = id; \
2583 encode_coding_emacs_mule (struct coding_system
*coding
)
2585 bool multibytep
= coding
->dst_multibyte
;
2586 int *charbuf
= coding
->charbuf
;
2587 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
2588 unsigned char *dst
= coding
->destination
+ coding
->produced
;
2589 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
2591 ptrdiff_t produced_chars
= 0;
2592 Lisp_Object attrs
, charset_list
;
2594 int preferred_charset_id
= -1;
2596 CODING_GET_INFO (coding
, attrs
, charset_list
);
2597 if (! EQ (charset_list
, Vemacs_mule_charset_list
))
2599 charset_list
= Vemacs_mule_charset_list
;
2600 ASET (attrs
, coding_attr_charset_list
, charset_list
);
2603 while (charbuf
< charbuf_end
)
2605 ASSURE_DESTINATION (safe_room
);
2610 /* Handle an annotation. */
2613 case CODING_ANNOTATE_COMPOSITION_MASK
:
2614 /* Not yet implemented. */
2616 case CODING_ANNOTATE_CHARSET_MASK
:
2617 preferred_charset_id
= charbuf
[3];
2618 if (preferred_charset_id
>= 0
2619 && NILP (Fmemq (make_number (preferred_charset_id
),
2621 preferred_charset_id
= -1;
2630 if (ASCII_CHAR_P (c
))
2631 EMIT_ONE_ASCII_BYTE (c
);
2632 else if (CHAR_BYTE8_P (c
))
2634 c
= CHAR_TO_BYTE8 (c
);
2639 struct charset
*charset
;
2643 unsigned char leading_codes
[2];
2645 if (preferred_charset_id
>= 0)
2649 charset
= CHARSET_FROM_ID (preferred_charset_id
);
2650 CODING_CHAR_CHARSET_P (coding
, dst
, dst_end
, c
, charset
, result
);
2652 code
= ENCODE_CHAR (charset
, c
);
2654 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
2658 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
2662 c
= coding
->default_char
;
2663 if (ASCII_CHAR_P (c
))
2665 EMIT_ONE_ASCII_BYTE (c
);
2668 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
2671 dimension
= CHARSET_DIMENSION (charset
);
2672 emacs_mule_id
= CHARSET_EMACS_MULE_ID (charset
);
2673 EMACS_MULE_LEADING_CODES (emacs_mule_id
, leading_codes
);
2674 EMIT_ONE_BYTE (leading_codes
[0]);
2675 if (leading_codes
[1])
2676 EMIT_ONE_BYTE (leading_codes
[1]);
2678 EMIT_ONE_BYTE (code
| 0x80);
2682 EMIT_ONE_BYTE (code
>> 8);
2683 EMIT_ONE_BYTE (code
& 0xFF);
2687 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
2688 coding
->produced_char
+= produced_chars
;
2689 coding
->produced
= dst
- coding
->destination
;
2694 /*** 7. ISO2022 handlers ***/
2696 /* The following note describes the coding system ISO2022 briefly.
2697 Since the intention of this note is to help understand the
2698 functions in this file, some parts are NOT ACCURATE or are OVERLY
2699 SIMPLIFIED. For thorough understanding, please refer to the
2700 original document of ISO2022. This is equivalent to the standard
2701 ECMA-35, obtainable from <URL:http://www.ecma.ch/> (*).
2703 ISO2022 provides many mechanisms to encode several character sets
2704 in 7-bit and 8-bit environments. For 7-bit environments, all text
2705 is encoded using bytes less than 128. This may make the encoded
2706 text a little bit longer, but the text passes more easily through
2707 several types of gateway, some of which strip off the MSB (Most
2710 There are two kinds of character sets: control character sets and
2711 graphic character sets. The former contain control characters such
2712 as `newline' and `escape' to provide control functions (control
2713 functions are also provided by escape sequences). The latter
2714 contain graphic characters such as 'A' and '-'. Emacs recognizes
2715 two control character sets and many graphic character sets.
2717 Graphic character sets are classified into one of the following
2718 four classes, according to the number of bytes (DIMENSION) and
2719 number of characters in one dimension (CHARS) of the set:
2720 - DIMENSION1_CHARS94
2721 - DIMENSION1_CHARS96
2722 - DIMENSION2_CHARS94
2723 - DIMENSION2_CHARS96
2725 In addition, each character set is assigned an identification tag,
2726 unique for each set, called the "final character" (denoted as <F>
2727 hereafter). The <F> of each character set is decided by ECMA(*)
2728 when it is registered in ISO. The code range of <F> is 0x30..0x7F
2729 (0x30..0x3F are for private use only).
2731 Note (*): ECMA = European Computer Manufacturers Association
2733 Here are examples of graphic character sets [NAME(<F>)]:
2734 o DIMENSION1_CHARS94 -- ASCII('B'), right-half-of-JISX0201('I'), ...
2735 o DIMENSION1_CHARS96 -- right-half-of-ISO8859-1('A'), ...
2736 o DIMENSION2_CHARS94 -- GB2312('A'), JISX0208('B'), ...
2737 o DIMENSION2_CHARS96 -- none for the moment
2739 A code area (1 byte=8 bits) is divided into 4 areas, C0, GL, C1, and GR.
2740 C0 [0x00..0x1F] -- control character plane 0
2741 GL [0x20..0x7F] -- graphic character plane 0
2742 C1 [0x80..0x9F] -- control character plane 1
2743 GR [0xA0..0xFF] -- graphic character plane 1
2745 A control character set is directly designated and invoked to C0 or
2746 C1 by an escape sequence. The most common case is that:
2747 - ISO646's control character set is designated/invoked to C0, and
2748 - ISO6429's control character set is designated/invoked to C1,
2749 and usually these designations/invocations are omitted in encoded
2750 text. In a 7-bit environment, only C0 can be used, and a control
2751 character for C1 is encoded by an appropriate escape sequence to
2752 fit into the environment. All control characters for C1 are
2753 defined to have corresponding escape sequences.
2755 A graphic character set is at first designated to one of four
2756 graphic registers (G0 through G3), then these graphic registers are
2757 invoked to GL or GR. These designations and invocations can be
2758 done independently. The most common case is that G0 is invoked to
2759 GL, G1 is invoked to GR, and ASCII is designated to G0. Usually
2760 these invocations and designations are omitted in encoded text.
2761 In a 7-bit environment, only GL can be used.
2763 When a graphic character set of CHARS94 is invoked to GL, codes
2764 0x20 and 0x7F of the GL area work as control characters SPACE and
2765 DEL respectively, and codes 0xA0 and 0xFF of the GR area should not
2768 There are two ways of invocation: locking-shift and single-shift.
2769 With locking-shift, the invocation lasts until the next different
2770 invocation, whereas with single-shift, the invocation affects the
2771 following character only and doesn't affect the locking-shift
2772 state. Invocations are done by the following control characters or
2775 ----------------------------------------------------------------------
2776 abbrev function cntrl escape seq description
2777 ----------------------------------------------------------------------
2778 SI/LS0 (shift-in) 0x0F none invoke G0 into GL
2779 SO/LS1 (shift-out) 0x0E none invoke G1 into GL
2780 LS2 (locking-shift-2) none ESC 'n' invoke G2 into GL
2781 LS3 (locking-shift-3) none ESC 'o' invoke G3 into GL
2782 LS1R (locking-shift-1 right) none ESC '~' invoke G1 into GR (*)
2783 LS2R (locking-shift-2 right) none ESC '}' invoke G2 into GR (*)
2784 LS3R (locking-shift 3 right) none ESC '|' invoke G3 into GR (*)
2785 SS2 (single-shift-2) 0x8E ESC 'N' invoke G2 for one char
2786 SS3 (single-shift-3) 0x8F ESC 'O' invoke G3 for one char
2787 ----------------------------------------------------------------------
2788 (*) These are not used by any known coding system.
2790 Control characters for these functions are defined by macros
2791 ISO_CODE_XXX in `coding.h'.
2793 Designations are done by the following escape sequences:
2794 ----------------------------------------------------------------------
2795 escape sequence description
2796 ----------------------------------------------------------------------
2797 ESC '(' <F> designate DIMENSION1_CHARS94<F> to G0
2798 ESC ')' <F> designate DIMENSION1_CHARS94<F> to G1
2799 ESC '*' <F> designate DIMENSION1_CHARS94<F> to G2
2800 ESC '+' <F> designate DIMENSION1_CHARS94<F> to G3
2801 ESC ',' <F> designate DIMENSION1_CHARS96<F> to G0 (*)
2802 ESC '-' <F> designate DIMENSION1_CHARS96<F> to G1
2803 ESC '.' <F> designate DIMENSION1_CHARS96<F> to G2
2804 ESC '/' <F> designate DIMENSION1_CHARS96<F> to G3
2805 ESC '$' '(' <F> designate DIMENSION2_CHARS94<F> to G0 (**)
2806 ESC '$' ')' <F> designate DIMENSION2_CHARS94<F> to G1
2807 ESC '$' '*' <F> designate DIMENSION2_CHARS94<F> to G2
2808 ESC '$' '+' <F> designate DIMENSION2_CHARS94<F> to G3
2809 ESC '$' ',' <F> designate DIMENSION2_CHARS96<F> to G0 (*)
2810 ESC '$' '-' <F> designate DIMENSION2_CHARS96<F> to G1
2811 ESC '$' '.' <F> designate DIMENSION2_CHARS96<F> to G2
2812 ESC '$' '/' <F> designate DIMENSION2_CHARS96<F> to G3
2813 ----------------------------------------------------------------------
2815 In this list, "DIMENSION1_CHARS94<F>" means a graphic character set
2816 of dimension 1, chars 94, and final character <F>, etc...
2818 Note (*): Although these designations are not allowed in ISO2022,
2819 Emacs accepts them on decoding, and produces them on encoding
2820 CHARS96 character sets in a coding system which is characterized as
2821 7-bit environment, non-locking-shift, and non-single-shift.
2823 Note (**): If <F> is '@', 'A', or 'B', the intermediate character
2824 '(' must be omitted. We refer to this as "short-form" hereafter.
2826 Now you may notice that there are a lot of ways of encoding the
2827 same multilingual text in ISO2022. Actually, there exist many
2828 coding systems such as Compound Text (used in X11's inter client
2829 communication, ISO-2022-JP (used in Japanese Internet), ISO-2022-KR
2830 (used in Korean Internet), EUC (Extended UNIX Code, used in Asian
2831 localized platforms), and all of these are variants of ISO2022.
2833 In addition to the above, Emacs handles two more kinds of escape
2834 sequences: ISO6429's direction specification and Emacs' private
2835 sequence for specifying character composition.
2837 ISO6429's direction specification takes the following form:
2838 o CSI ']' -- end of the current direction
2839 o CSI '0' ']' -- end of the current direction
2840 o CSI '1' ']' -- start of left-to-right text
2841 o CSI '2' ']' -- start of right-to-left text
2842 The control character CSI (0x9B: control sequence introducer) is
2843 abbreviated to the escape sequence ESC '[' in a 7-bit environment.
2845 Character composition specification takes the following form:
2846 o ESC '0' -- start relative composition
2847 o ESC '1' -- end composition
2848 o ESC '2' -- start rule-base composition (*)
2849 o ESC '3' -- start relative composition with alternate chars (**)
2850 o ESC '4' -- start rule-base composition with alternate chars (**)
2851 Since these are not standard escape sequences of any ISO standard,
2852 the use of them with these meanings is restricted to Emacs only.
2854 (*) This form is used only in Emacs 20.7 and older versions,
2855 but newer versions can safely decode it.
2856 (**) This form is used only in Emacs 21.1 and newer versions,
2857 and older versions can't decode it.
2859 Here's a list of example usages of these composition escape
2860 sequences (categorized by `enum composition_method').
2862 COMPOSITION_RELATIVE:
2863 ESC 0 CHAR [ CHAR ] ESC 1
2864 COMPOSITION_WITH_RULE:
2865 ESC 2 CHAR [ RULE CHAR ] ESC 1
2866 COMPOSITION_WITH_ALTCHARS:
2867 ESC 3 ALTCHAR [ ALTCHAR ] ESC 0 CHAR [ CHAR ] ESC 1
2868 COMPOSITION_WITH_RULE_ALTCHARS:
2869 ESC 4 ALTCHAR [ RULE ALTCHAR ] ESC 0 CHAR [ CHAR ] ESC 1 */
2871 static enum iso_code_class_type iso_code_class
[256];
2873 #define SAFE_CHARSET_P(coding, id) \
2874 ((id) <= (coding)->max_charset_id \
2875 && (coding)->safe_charsets[id] != 255)
2878 setup_iso_safe_charsets (Lisp_Object attrs
)
2880 Lisp_Object charset_list
, safe_charsets
;
2881 Lisp_Object request
;
2882 Lisp_Object reg_usage
;
2884 EMACS_INT reg94
, reg96
;
2885 int flags
= XINT (AREF (attrs
, coding_attr_iso_flags
));
2888 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
2889 if ((flags
& CODING_ISO_FLAG_FULL_SUPPORT
)
2890 && ! EQ (charset_list
, Viso_2022_charset_list
))
2892 charset_list
= Viso_2022_charset_list
;
2893 ASET (attrs
, coding_attr_charset_list
, charset_list
);
2894 ASET (attrs
, coding_attr_safe_charsets
, Qnil
);
2897 if (STRINGP (AREF (attrs
, coding_attr_safe_charsets
)))
2901 for (tail
= charset_list
; CONSP (tail
); tail
= XCDR (tail
))
2903 int id
= XINT (XCAR (tail
));
2904 if (max_charset_id
< id
)
2905 max_charset_id
= id
;
2908 safe_charsets
= make_uninit_string (max_charset_id
+ 1);
2909 memset (SDATA (safe_charsets
), 255, max_charset_id
+ 1);
2910 request
= AREF (attrs
, coding_attr_iso_request
);
2911 reg_usage
= AREF (attrs
, coding_attr_iso_usage
);
2912 reg94
= XINT (XCAR (reg_usage
));
2913 reg96
= XINT (XCDR (reg_usage
));
2915 for (tail
= charset_list
; CONSP (tail
); tail
= XCDR (tail
))
2919 struct charset
*charset
;
2922 charset
= CHARSET_FROM_ID (XINT (id
));
2923 reg
= Fcdr (Fassq (id
, request
));
2925 SSET (safe_charsets
, XINT (id
), XINT (reg
));
2926 else if (charset
->iso_chars_96
)
2929 SSET (safe_charsets
, XINT (id
), reg96
);
2934 SSET (safe_charsets
, XINT (id
), reg94
);
2937 ASET (attrs
, coding_attr_safe_charsets
, safe_charsets
);
2941 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
2942 Return true if a text is encoded in one of ISO-2022 based coding
2946 detect_coding_iso_2022 (struct coding_system
*coding
,
2947 struct coding_detection_info
*detect_info
)
2949 const unsigned char *src
= coding
->source
, *src_base
= src
;
2950 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
2951 bool multibytep
= coding
->src_multibyte
;
2952 bool single_shifting
= 0;
2955 ptrdiff_t consumed_chars
= 0;
2959 int composition_count
= -1;
2961 detect_info
->checked
|= CATEGORY_MASK_ISO
;
2963 for (i
= coding_category_iso_7
; i
<= coding_category_iso_8_else
; i
++)
2965 struct coding_system
*this = &(coding_categories
[i
]);
2966 Lisp_Object attrs
, val
;
2970 attrs
= CODING_ID_ATTRS (this->id
);
2971 if (CODING_ISO_FLAGS (this) & CODING_ISO_FLAG_FULL_SUPPORT
2972 && ! EQ (CODING_ATTR_CHARSET_LIST (attrs
), Viso_2022_charset_list
))
2973 setup_iso_safe_charsets (attrs
);
2974 val
= CODING_ATTR_SAFE_CHARSETS (attrs
);
2975 this->max_charset_id
= SCHARS (val
) - 1;
2976 this->safe_charsets
= SDATA (val
);
2979 /* A coding system of this category is always ASCII compatible. */
2980 src
+= coding
->head_ascii
;
2982 while (rejected
!= CATEGORY_MASK_ISO
)
2989 if (inhibit_iso_escape_detection
)
2991 single_shifting
= 0;
2993 if (c
== 'N' || c
== 'O')
2995 /* ESC <Fe> for SS2 or SS3. */
2996 single_shifting
= 1;
2997 rejected
|= CATEGORY_MASK_ISO_7BIT
| CATEGORY_MASK_ISO_8BIT
;
3001 /* End of composition. */
3002 if (composition_count
< 0
3003 || composition_count
> MAX_COMPOSITION_COMPONENTS
)
3006 composition_count
= -1;
3007 found
|= CATEGORY_MASK_ISO
;
3009 else if (c
>= '0' && c
<= '4')
3011 /* ESC <Fp> for start/end composition. */
3012 composition_count
= 0;
3016 if (c
>= '(' && c
<= '/')
3018 /* Designation sequence for a charset of dimension 1. */
3020 if (c1
< ' ' || c1
>= 0x80
3021 || (id
= iso_charset_table
[0][c
>= ','][c1
]) < 0)
3022 /* Invalid designation sequence. Just ignore. */
3027 /* Designation sequence for a charset of dimension 2. */
3029 if (c
>= '@' && c
<= 'B')
3030 /* Designation for JISX0208.1978, GB2312, or JISX0208. */
3031 id
= iso_charset_table
[1][0][c
];
3032 else if (c
>= '(' && c
<= '/')
3035 if (c1
< ' ' || c1
>= 0x80
3036 || (id
= iso_charset_table
[1][c
>= ','][c1
]) < 0)
3037 /* Invalid designation sequence. Just ignore. */
3041 /* Invalid designation sequence. Just ignore it. */
3046 /* Invalid escape sequence. Just ignore it. */
3050 /* We found a valid designation sequence for CHARSET. */
3051 rejected
|= CATEGORY_MASK_ISO_8BIT
;
3052 if (SAFE_CHARSET_P (&coding_categories
[coding_category_iso_7
],
3054 found
|= CATEGORY_MASK_ISO_7
;
3056 rejected
|= CATEGORY_MASK_ISO_7
;
3057 if (SAFE_CHARSET_P (&coding_categories
[coding_category_iso_7_tight
],
3059 found
|= CATEGORY_MASK_ISO_7_TIGHT
;
3061 rejected
|= CATEGORY_MASK_ISO_7_TIGHT
;
3062 if (SAFE_CHARSET_P (&coding_categories
[coding_category_iso_7_else
],
3064 found
|= CATEGORY_MASK_ISO_7_ELSE
;
3066 rejected
|= CATEGORY_MASK_ISO_7_ELSE
;
3067 if (SAFE_CHARSET_P (&coding_categories
[coding_category_iso_8_else
],
3069 found
|= CATEGORY_MASK_ISO_8_ELSE
;
3071 rejected
|= CATEGORY_MASK_ISO_8_ELSE
;
3077 /* Locking shift out/in. */
3078 if (inhibit_iso_escape_detection
)
3080 single_shifting
= 0;
3081 rejected
|= CATEGORY_MASK_ISO_7BIT
| CATEGORY_MASK_ISO_8BIT
;
3085 /* Control sequence introducer. */
3086 single_shifting
= 0;
3087 rejected
|= CATEGORY_MASK_ISO_7BIT
| CATEGORY_MASK_ISO_7_ELSE
;
3088 found
|= CATEGORY_MASK_ISO_8_ELSE
;
3089 goto check_extra_latin
;
3094 if (inhibit_iso_escape_detection
)
3096 single_shifting
= 0;
3097 rejected
|= CATEGORY_MASK_ISO_7BIT
;
3098 if (CODING_ISO_FLAGS (&coding_categories
[coding_category_iso_8_1
])
3099 & CODING_ISO_FLAG_SINGLE_SHIFT
)
3101 found
|= CATEGORY_MASK_ISO_8_1
;
3102 single_shifting
= 1;
3104 if (CODING_ISO_FLAGS (&coding_categories
[coding_category_iso_8_2
])
3105 & CODING_ISO_FLAG_SINGLE_SHIFT
)
3107 found
|= CATEGORY_MASK_ISO_8_2
;
3108 single_shifting
= 1;
3110 if (single_shifting
)
3112 goto check_extra_latin
;
3119 if (composition_count
>= 0)
3120 composition_count
++;
3121 single_shifting
= 0;
3126 rejected
|= CATEGORY_MASK_ISO_7BIT
| CATEGORY_MASK_ISO_7_ELSE
;
3127 found
|= CATEGORY_MASK_ISO_8_1
;
3128 /* Check the length of succeeding codes of the range
3129 0xA0..0FF. If the byte length is even, we include
3130 CATEGORY_MASK_ISO_8_2 in `found'. We can check this
3131 only when we are not single shifting. */
3132 if (! single_shifting
3133 && ! (rejected
& CATEGORY_MASK_ISO_8_2
))
3136 while (src
< src_end
)
3148 if (len
& 1 && src
< src_end
)
3150 rejected
|= CATEGORY_MASK_ISO_8_2
;
3151 if (composition_count
>= 0)
3152 composition_count
+= len
;
3156 found
|= CATEGORY_MASK_ISO_8_2
;
3157 if (composition_count
>= 0)
3158 composition_count
+= len
/ 2;
3164 if (! VECTORP (Vlatin_extra_code_table
)
3165 || NILP (AREF (Vlatin_extra_code_table
, c
)))
3167 rejected
= CATEGORY_MASK_ISO
;
3170 if (CODING_ISO_FLAGS (&coding_categories
[coding_category_iso_8_1
])
3171 & CODING_ISO_FLAG_LATIN_EXTRA
)
3172 found
|= CATEGORY_MASK_ISO_8_1
;
3174 rejected
|= CATEGORY_MASK_ISO_8_1
;
3175 rejected
|= CATEGORY_MASK_ISO_8_2
;
3179 detect_info
->rejected
|= CATEGORY_MASK_ISO
;
3183 detect_info
->rejected
|= rejected
;
3184 detect_info
->found
|= (found
& ~rejected
);
3189 /* Set designation state into CODING. Set CHARS_96 to -1 if the
3190 escape sequence should be kept. */
3191 #define DECODE_DESIGNATION(reg, dim, chars_96, final) \
3195 if (final < '0' || final >= 128 \
3196 || ((id = ISO_CHARSET_TABLE (dim, chars_96, final)) < 0) \
3197 || !SAFE_CHARSET_P (coding, id)) \
3199 CODING_ISO_DESIGNATION (coding, reg) = -2; \
3203 prev = CODING_ISO_DESIGNATION (coding, reg); \
3204 if (id == charset_jisx0201_roman) \
3206 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_USE_ROMAN) \
3207 id = charset_ascii; \
3209 else if (id == charset_jisx0208_1978) \
3211 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_USE_OLDJIS) \
3212 id = charset_jisx0208; \
3214 CODING_ISO_DESIGNATION (coding, reg) = id; \
3215 /* If there was an invalid designation to REG previously, and this \
3216 designation is ASCII to REG, we should keep this designation \
3218 if (prev == -2 && id == charset_ascii) \
3223 /* Handle these composition sequence (ALT: alternate char):
3225 (1) relative composition: ESC 0 CHAR ... ESC 1
3226 (2) rulebase composition: ESC 2 CHAR RULE CHAR RULE ... CHAR ESC 1
3227 (3) altchar composition: ESC 3 ALT ... ALT ESC 0 CHAR ... ESC 1
3228 (4) alt&rule composition: ESC 4 ALT RULE ... ALT ESC 0 CHAR ... ESC 1
3230 When the start sequence (ESC 0/2/3/4) is found, this annotation
3233 [ -LENGTH(==-5) CODING_ANNOTATE_COMPOSITION_MASK NCHARS(==0) 0 METHOD ]
3235 Then, upon reading CHAR or RULE (one or two bytes), these codes are
3236 produced until the end sequence (ESC 1) is found:
3239 (2) CHAR -2 DECODED-RULE CHAR -2 DECODED-RULE ... CHAR
3240 (3) ALT ... ALT -1 -1 CHAR ... CHAR
3241 (4) ALT -2 DECODED-RULE ALT -2 DECODED-RULE ... ALT -1 -1 CHAR ... CHAR
3243 When the end sequence (ESC 1) is found, LENGTH and NCHARS in the
3244 annotation header is updated as below:
3246 (1) LENGTH: unchanged, NCHARS: number of CHARs
3247 (2) LENGTH: unchanged, NCHARS: number of CHARs
3248 (3) LENGTH: += number of ALTs + 2, NCHARS: number of CHARs
3249 (4) LENGTH: += number of ALTs * 3, NCHARS: number of CHARs
3251 If an error is found while composing, the annotation header is
3254 [ ESC '0'/'2'/'3'/'4' -2 0 ]
3256 and the sequence [ -2 DECODED-RULE ] is changed to the original
3257 byte sequence as below:
3258 o the original byte sequence is B: [ B -1 ]
3259 o the original byte sequence is B1 B2: [ B1 B2 ]
3260 and the sequence [ -1 -1 ] is changed to the original byte
3265 /* Decode a composition rule C1 and maybe one more byte from the
3266 source, and set RULE to the encoded composition rule. If the rule
3267 is invalid, goto invalid_code. */
3269 #define DECODE_COMPOSITION_RULE(rule) \
3273 goto invalid_code; \
3274 if (rule < 81) /* old format (before ver.21) */ \
3276 int gref = (rule) / 9; \
3277 int nref = (rule) % 9; \
3278 if (gref == 4) gref = 10; \
3279 if (nref == 4) nref = 10; \
3280 rule = COMPOSITION_ENCODE_RULE (gref, nref); \
3282 else /* new format (after ver.21) */ \
3286 ONE_MORE_BYTE (b); \
3287 if (! COMPOSITION_ENCODE_RULE_VALID (rule - 81, b - 32)) \
3288 goto invalid_code; \
3289 rule = COMPOSITION_ENCODE_RULE (rule - 81, b - 32); \
3290 rule += 0x100; /* Distinguish it from the old format. */ \
3294 #define ENCODE_COMPOSITION_RULE(rule) \
3296 int gref = (rule % 0x100) / 12, nref = (rule % 0x100) % 12; \
3298 if (rule < 0x100) /* old format */ \
3300 if (gref == 10) gref = 4; \
3301 if (nref == 10) nref = 4; \
3302 charbuf[idx] = 32 + gref * 9 + nref; \
3303 charbuf[idx + 1] = -1; \
3306 else /* new format */ \
3308 charbuf[idx] = 32 + 81 + gref; \
3309 charbuf[idx + 1] = 32 + nref; \
3314 /* Finish the current composition as invalid. */
3317 finish_composition (int *charbuf
, struct composition_status
*cmp_status
)
3319 int idx
= - cmp_status
->length
;
3322 /* Recover the original ESC sequence */
3323 charbuf
[idx
++] = ISO_CODE_ESC
;
3324 charbuf
[idx
++] = (cmp_status
->method
== COMPOSITION_RELATIVE
? '0'
3325 : cmp_status
->method
== COMPOSITION_WITH_RULE
? '2'
3326 : cmp_status
->method
== COMPOSITION_WITH_ALTCHARS
? '3'
3327 /* cmp_status->method == COMPOSITION_WITH_RULE_ALTCHARS */
3329 charbuf
[idx
++] = -2;
3331 charbuf
[idx
++] = -1;
3332 new_chars
= cmp_status
->nchars
;
3333 if (cmp_status
->method
>= COMPOSITION_WITH_RULE
)
3334 for (; idx
< 0; idx
++)
3336 int elt
= charbuf
[idx
];
3340 ENCODE_COMPOSITION_RULE (charbuf
[idx
+ 1]);
3345 charbuf
[idx
++] = ISO_CODE_ESC
;
3350 cmp_status
->state
= COMPOSING_NO
;
3354 /* If characters are under composition, finish the composition. */
3355 #define MAYBE_FINISH_COMPOSITION() \
3357 if (cmp_status->state != COMPOSING_NO) \
3358 char_offset += finish_composition (charbuf, cmp_status); \
3361 /* Handle composition start sequence ESC 0, ESC 2, ESC 3, or ESC 4.
3363 ESC 0 : relative composition : ESC 0 CHAR ... ESC 1
3364 ESC 2 : rulebase composition : ESC 2 CHAR RULE CHAR RULE ... CHAR ESC 1
3365 ESC 3 : altchar composition : ESC 3 CHAR ... ESC 0 CHAR ... ESC 1
3366 ESC 4 : alt&rule composition : ESC 4 CHAR RULE ... CHAR ESC 0 CHAR ... ESC 1
3368 Produce this annotation sequence now:
3370 [ -LENGTH(==-4) CODING_ANNOTATE_COMPOSITION_MASK NCHARS(==0) METHOD ]
3373 #define DECODE_COMPOSITION_START(c1) \
3376 && ((cmp_status->state == COMPOSING_COMPONENT_CHAR \
3377 && cmp_status->method == COMPOSITION_WITH_ALTCHARS) \
3378 || (cmp_status->state == COMPOSING_COMPONENT_RULE \
3379 && cmp_status->method == COMPOSITION_WITH_RULE_ALTCHARS))) \
3383 cmp_status->state = COMPOSING_CHAR; \
3384 cmp_status->length += 2; \
3388 MAYBE_FINISH_COMPOSITION (); \
3389 cmp_status->method = (c1 == '0' ? COMPOSITION_RELATIVE \
3390 : c1 == '2' ? COMPOSITION_WITH_RULE \
3391 : c1 == '3' ? COMPOSITION_WITH_ALTCHARS \
3392 : COMPOSITION_WITH_RULE_ALTCHARS); \
3394 = (c1 <= '2' ? COMPOSING_CHAR : COMPOSING_COMPONENT_CHAR); \
3395 ADD_COMPOSITION_DATA (charbuf, 0, 0, cmp_status->method); \
3396 cmp_status->length = MAX_ANNOTATION_LENGTH; \
3397 cmp_status->nchars = cmp_status->ncomps = 0; \
3398 coding->annotated = 1; \
3403 /* Handle composition end sequence ESC 1. */
3405 #define DECODE_COMPOSITION_END() \
3407 if (cmp_status->nchars == 0 \
3408 || ((cmp_status->state == COMPOSING_CHAR) \
3409 == (cmp_status->method == COMPOSITION_WITH_RULE))) \
3411 MAYBE_FINISH_COMPOSITION (); \
3412 goto invalid_code; \
3414 if (cmp_status->method == COMPOSITION_WITH_ALTCHARS) \
3415 charbuf[- cmp_status->length] -= cmp_status->ncomps + 2; \
3416 else if (cmp_status->method == COMPOSITION_WITH_RULE_ALTCHARS) \
3417 charbuf[- cmp_status->length] -= cmp_status->ncomps * 3; \
3418 charbuf[- cmp_status->length + 2] = cmp_status->nchars; \
3419 char_offset += cmp_status->nchars; \
3420 cmp_status->state = COMPOSING_NO; \
3423 /* Store a composition rule RULE in charbuf, and update cmp_status. */
3425 #define STORE_COMPOSITION_RULE(rule) \
3428 *charbuf++ = rule; \
3429 cmp_status->length += 2; \
3430 cmp_status->state--; \
3433 /* Store a composed char or a component char C in charbuf, and update
3436 #define STORE_COMPOSITION_CHAR(c) \
3439 cmp_status->length++; \
3440 if (cmp_status->state == COMPOSING_CHAR) \
3441 cmp_status->nchars++; \
3443 cmp_status->ncomps++; \
3444 if (cmp_status->method == COMPOSITION_WITH_RULE \
3445 || (cmp_status->method == COMPOSITION_WITH_RULE_ALTCHARS \
3446 && cmp_status->state == COMPOSING_COMPONENT_CHAR)) \
3447 cmp_status->state++; \
3451 /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions". */
3454 decode_coding_iso_2022 (struct coding_system
*coding
)
3456 const unsigned char *src
= coding
->source
+ coding
->consumed
;
3457 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
3458 const unsigned char *src_base
;
3459 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
3460 /* We may produce two annotations (charset and composition) in one
3461 loop and one more charset annotation at the end. */
3463 = coding
->charbuf
+ coding
->charbuf_size
- (MAX_ANNOTATION_LENGTH
* 3);
3464 ptrdiff_t consumed_chars
= 0, consumed_chars_base
;
3465 bool multibytep
= coding
->src_multibyte
;
3466 /* Charsets invoked to graphic plane 0 and 1 respectively. */
3467 int charset_id_0
= CODING_ISO_INVOKED_CHARSET (coding
, 0);
3468 int charset_id_1
= CODING_ISO_INVOKED_CHARSET (coding
, 1);
3469 int charset_id_2
, charset_id_3
;
3470 struct charset
*charset
;
3472 struct composition_status
*cmp_status
= CODING_ISO_CMP_STATUS (coding
);
3473 Lisp_Object attrs
= CODING_ID_ATTRS (coding
->id
);
3474 ptrdiff_t char_offset
= coding
->produced_char
;
3475 ptrdiff_t last_offset
= char_offset
;
3476 int last_id
= charset_ascii
;
3478 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
3479 int byte_after_cr
= -1;
3482 setup_iso_safe_charsets (attrs
);
3483 coding
->safe_charsets
= SDATA (CODING_ATTR_SAFE_CHARSETS (attrs
));
3485 if (cmp_status
->state
!= COMPOSING_NO
)
3487 if (charbuf_end
- charbuf
< cmp_status
->length
)
3489 for (i
= 0; i
< cmp_status
->length
; i
++)
3490 *charbuf
++ = cmp_status
->carryover
[i
];
3491 coding
->annotated
= 1;
3499 consumed_chars_base
= consumed_chars
;
3501 if (charbuf
>= charbuf_end
)
3503 if (byte_after_cr
>= 0)
3508 if (byte_after_cr
>= 0)
3509 c1
= byte_after_cr
, byte_after_cr
= -1;
3515 if (CODING_ISO_EXTSEGMENT_LEN (coding
) > 0)
3517 *charbuf
++ = ASCII_BYTE_P (c1
) ? c1
: BYTE8_TO_CHAR (c1
);
3519 CODING_ISO_EXTSEGMENT_LEN (coding
)--;
3523 if (CODING_ISO_EMBEDDED_UTF_8 (coding
))
3525 if (c1
== ISO_CODE_ESC
)
3527 if (src
+ 1 >= src_end
)
3528 goto no_more_source
;
3529 *charbuf
++ = ISO_CODE_ESC
;
3531 if (src
[0] == '%' && src
[1] == '@')
3534 consumed_chars
+= 2;
3536 /* We are sure charbuf can contain two more chars. */
3539 CODING_ISO_EMBEDDED_UTF_8 (coding
) = 0;
3544 *charbuf
++ = ASCII_BYTE_P (c1
) ? c1
: BYTE8_TO_CHAR (c1
);
3550 if ((cmp_status
->state
== COMPOSING_RULE
3551 || cmp_status
->state
== COMPOSING_COMPONENT_RULE
)
3552 && c1
!= ISO_CODE_ESC
)
3556 DECODE_COMPOSITION_RULE (rule
);
3557 STORE_COMPOSITION_RULE (rule
);
3561 /* We produce at most one character. */
3562 switch (iso_code_class
[c1
])
3564 case ISO_0x20_or_0x7F
:
3565 if (charset_id_0
< 0
3566 || ! CHARSET_ISO_CHARS_96 (CHARSET_FROM_ID (charset_id_0
)))
3567 /* This is SPACE or DEL. */
3568 charset
= CHARSET_FROM_ID (charset_ascii
);
3570 charset
= CHARSET_FROM_ID (charset_id_0
);
3573 case ISO_graphic_plane_0
:
3574 if (charset_id_0
< 0)
3575 charset
= CHARSET_FROM_ID (charset_ascii
);
3577 charset
= CHARSET_FROM_ID (charset_id_0
);
3580 case ISO_0xA0_or_0xFF
:
3581 if (charset_id_1
< 0
3582 || ! CHARSET_ISO_CHARS_96 (CHARSET_FROM_ID (charset_id_1
))
3583 || CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SEVEN_BITS
)
3585 /* This is a graphic character, we fall down ... */
3587 case ISO_graphic_plane_1
:
3588 if (charset_id_1
< 0)
3590 charset
= CHARSET_FROM_ID (charset_id_1
);
3594 if (eol_dos
&& c1
== '\r')
3595 ONE_MORE_BYTE (byte_after_cr
);
3596 MAYBE_FINISH_COMPOSITION ();
3597 charset
= CHARSET_FROM_ID (charset_ascii
);
3604 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_LOCKING_SHIFT
)
3605 || CODING_ISO_DESIGNATION (coding
, 1) < 0)
3607 CODING_ISO_INVOCATION (coding
, 0) = 1;
3608 charset_id_0
= CODING_ISO_INVOKED_CHARSET (coding
, 0);
3612 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_LOCKING_SHIFT
))
3614 CODING_ISO_INVOCATION (coding
, 0) = 0;
3615 charset_id_0
= CODING_ISO_INVOKED_CHARSET (coding
, 0);
3618 case ISO_single_shift_2_7
:
3619 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SEVEN_BITS
))
3621 case ISO_single_shift_2
:
3622 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SINGLE_SHIFT
))
3624 /* SS2 is handled as an escape sequence of ESC 'N' */
3626 goto label_escape_sequence
;
3628 case ISO_single_shift_3
:
3629 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SINGLE_SHIFT
))
3631 /* SS2 is handled as an escape sequence of ESC 'O' */
3633 goto label_escape_sequence
;
3635 case ISO_control_sequence_introducer
:
3636 /* CSI is handled as an escape sequence of ESC '[' ... */
3638 goto label_escape_sequence
;
3642 label_escape_sequence
:
3643 /* Escape sequences handled here are invocation,
3644 designation, direction specification, and character
3645 composition specification. */
3648 case '&': /* revision of following character set */
3650 if (!(c1
>= '@' && c1
<= '~'))
3653 if (c1
!= ISO_CODE_ESC
)
3656 goto label_escape_sequence
;
3658 case '$': /* designation of 2-byte character set */
3659 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_DESIGNATION
))
3665 if (c1
>= '@' && c1
<= 'B')
3666 { /* designation of JISX0208.1978, GB2312.1980,
3668 reg
= 0, chars96
= 0;
3670 else if (c1
>= 0x28 && c1
<= 0x2B)
3671 { /* designation of DIMENSION2_CHARS94 character set */
3672 reg
= c1
- 0x28, chars96
= 0;
3675 else if (c1
>= 0x2C && c1
<= 0x2F)
3676 { /* designation of DIMENSION2_CHARS96 character set */
3677 reg
= c1
- 0x2C, chars96
= 1;
3682 DECODE_DESIGNATION (reg
, 2, chars96
, c1
);
3683 /* We must update these variables now. */
3685 charset_id_0
= CODING_ISO_INVOKED_CHARSET (coding
, 0);
3687 charset_id_1
= CODING_ISO_INVOKED_CHARSET (coding
, 1);
3693 case 'n': /* invocation of locking-shift-2 */
3694 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_LOCKING_SHIFT
)
3695 || CODING_ISO_DESIGNATION (coding
, 2) < 0)
3697 CODING_ISO_INVOCATION (coding
, 0) = 2;
3698 charset_id_0
= CODING_ISO_INVOKED_CHARSET (coding
, 0);
3701 case 'o': /* invocation of locking-shift-3 */
3702 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_LOCKING_SHIFT
)
3703 || CODING_ISO_DESIGNATION (coding
, 3) < 0)
3705 CODING_ISO_INVOCATION (coding
, 0) = 3;
3706 charset_id_0
= CODING_ISO_INVOKED_CHARSET (coding
, 0);
3709 case 'N': /* invocation of single-shift-2 */
3710 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SINGLE_SHIFT
)
3711 || CODING_ISO_DESIGNATION (coding
, 2) < 0)
3713 charset_id_2
= CODING_ISO_DESIGNATION (coding
, 2);
3714 if (charset_id_2
< 0)
3715 charset
= CHARSET_FROM_ID (charset_ascii
);
3717 charset
= CHARSET_FROM_ID (charset_id_2
);
3719 if (c1
< 0x20 || (c1
>= 0x80 && c1
< 0xA0))
3723 case 'O': /* invocation of single-shift-3 */
3724 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SINGLE_SHIFT
)
3725 || CODING_ISO_DESIGNATION (coding
, 3) < 0)
3727 charset_id_3
= CODING_ISO_DESIGNATION (coding
, 3);
3728 if (charset_id_3
< 0)
3729 charset
= CHARSET_FROM_ID (charset_ascii
);
3731 charset
= CHARSET_FROM_ID (charset_id_3
);
3733 if (c1
< 0x20 || (c1
>= 0x80 && c1
< 0xA0))
3737 case '0': case '2': case '3': case '4': /* start composition */
3738 if (! (coding
->common_flags
& CODING_ANNOTATE_COMPOSITION_MASK
))
3740 if (last_id
!= charset_ascii
)
3742 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
3743 last_id
= charset_ascii
;
3744 last_offset
= char_offset
;
3746 DECODE_COMPOSITION_START (c1
);
3749 case '1': /* end composition */
3750 if (cmp_status
->state
== COMPOSING_NO
)
3752 DECODE_COMPOSITION_END ();
3755 case '[': /* specification of direction */
3756 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_DIRECTION
))
3758 /* For the moment, nested direction is not supported.
3759 So, `coding->mode & CODING_MODE_DIRECTION' zero means
3760 left-to-right, and nonzero means right-to-left. */
3764 case ']': /* end of the current direction */
3765 coding
->mode
&= ~CODING_MODE_DIRECTION
;
3767 case '0': /* end of the current direction */
3768 case '1': /* start of left-to-right direction */
3771 coding
->mode
&= ~CODING_MODE_DIRECTION
;
3776 case '2': /* start of right-to-left direction */
3779 coding
->mode
|= CODING_MODE_DIRECTION
;
3793 /* CTEXT extended segment:
3794 ESC % / [0-4] M L --ENCODING-NAME-- \002 --BYTES--
3795 We keep these bytes as is for the moment.
3796 They may be decoded by post-read-conversion. */
3800 ONE_MORE_BYTE (dim
);
3801 if (dim
< '0' || dim
> '4')
3809 size
= ((M
- 128) * 128) + (L
- 128);
3810 if (charbuf
+ 6 > charbuf_end
)
3812 *charbuf
++ = ISO_CODE_ESC
;
3816 *charbuf
++ = BYTE8_TO_CHAR (M
);
3817 *charbuf
++ = BYTE8_TO_CHAR (L
);
3818 CODING_ISO_EXTSEGMENT_LEN (coding
) = size
;
3822 /* XFree86 extension for embedding UTF-8 in CTEXT:
3823 ESC % G --UTF-8-BYTES-- ESC % @
3824 We keep these bytes as is for the moment.
3825 They may be decoded by post-read-conversion. */
3826 if (charbuf
+ 3 > charbuf_end
)
3828 *charbuf
++ = ISO_CODE_ESC
;
3831 CODING_ISO_EMBEDDED_UTF_8 (coding
) = 1;
3839 if (! (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_DESIGNATION
))
3844 if (c1
>= 0x28 && c1
<= 0x2B)
3845 { /* designation of DIMENSION1_CHARS94 character set */
3846 reg
= c1
- 0x28, chars96
= 0;
3849 else if (c1
>= 0x2C && c1
<= 0x2F)
3850 { /* designation of DIMENSION1_CHARS96 character set */
3851 reg
= c1
- 0x2C, chars96
= 1;
3856 DECODE_DESIGNATION (reg
, 1, chars96
, c1
);
3857 /* We must update these variables now. */
3859 charset_id_0
= CODING_ISO_INVOKED_CHARSET (coding
, 0);
3861 charset_id_1
= CODING_ISO_INVOKED_CHARSET (coding
, 1);
3873 if (cmp_status
->state
== COMPOSING_NO
3874 && charset
->id
!= charset_ascii
3875 && last_id
!= charset
->id
)
3877 if (last_id
!= charset_ascii
)
3878 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
3879 last_id
= charset
->id
;
3880 last_offset
= char_offset
;
3883 /* Now we know CHARSET and 1st position code C1 of a character.
3884 Produce a decoded character while getting 2nd and 3rd
3885 position codes C2, C3 if necessary. */
3886 if (CHARSET_DIMENSION (charset
) > 1)
3889 if (c2
< 0x20 || (c2
>= 0x80 && c2
< 0xA0)
3890 || ((c1
& 0x80) != (c2
& 0x80)))
3891 /* C2 is not in a valid range. */
3893 if (CHARSET_DIMENSION (charset
) == 2)
3894 c1
= (c1
<< 8) | c2
;
3898 if (c3
< 0x20 || (c3
>= 0x80 && c3
< 0xA0)
3899 || ((c1
& 0x80) != (c3
& 0x80)))
3900 /* C3 is not in a valid range. */
3902 c1
= (c1
<< 16) | (c2
<< 8) | c2
;
3906 CODING_DECODE_CHAR (coding
, src
, src_base
, src_end
, charset
, c1
, c
);
3909 MAYBE_FINISH_COMPOSITION ();
3910 for (; src_base
< src
; src_base
++, char_offset
++)
3912 if (ASCII_BYTE_P (*src_base
))
3913 *charbuf
++ = *src_base
;
3915 *charbuf
++ = BYTE8_TO_CHAR (*src_base
);
3918 else if (cmp_status
->state
== COMPOSING_NO
)
3923 else if ((cmp_status
->state
== COMPOSING_CHAR
3924 ? cmp_status
->nchars
3925 : cmp_status
->ncomps
)
3926 >= MAX_COMPOSITION_COMPONENTS
)
3928 /* Too long composition. */
3929 MAYBE_FINISH_COMPOSITION ();
3934 STORE_COMPOSITION_CHAR (c
);
3938 MAYBE_FINISH_COMPOSITION ();
3940 consumed_chars
= consumed_chars_base
;
3942 *charbuf
++ = c
< 0 ? -c
: ASCII_BYTE_P (c
) ? c
: BYTE8_TO_CHAR (c
);
3945 /* Reset the invocation and designation status to the safest
3946 one; i.e. designate ASCII to the graphic register 0, and
3947 invoke that register to the graphic plane 0. This typically
3948 helps the case that an designation sequence for ASCII "ESC (
3949 B" is somehow broken (e.g. broken by a newline). */
3950 CODING_ISO_INVOCATION (coding
, 0) = 0;
3951 CODING_ISO_DESIGNATION (coding
, 0) = charset_ascii
;
3952 charset_id_0
= charset_ascii
;
3960 if (cmp_status
->state
!= COMPOSING_NO
)
3962 if (coding
->mode
& CODING_MODE_LAST_BLOCK
)
3963 MAYBE_FINISH_COMPOSITION ();
3966 charbuf
-= cmp_status
->length
;
3967 for (i
= 0; i
< cmp_status
->length
; i
++)
3968 cmp_status
->carryover
[i
] = charbuf
[i
];
3971 else if (last_id
!= charset_ascii
)
3972 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
3973 coding
->consumed_char
+= consumed_chars_base
;
3974 coding
->consumed
= src_base
- coding
->source
;
3975 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
3979 /* ISO2022 encoding stuff. */
3982 It is not enough to say just "ISO2022" on encoding, we have to
3983 specify more details. In Emacs, each coding system of ISO2022
3984 variant has the following specifications:
3985 1. Initial designation to G0 thru G3.
3986 2. Allows short-form designation?
3987 3. ASCII should be designated to G0 before control characters?
3988 4. ASCII should be designated to G0 at end of line?
3989 5. 7-bit environment or 8-bit environment?
3990 6. Use locking-shift?
3991 7. Use Single-shift?
3992 And the following two are only for Japanese:
3993 8. Use ASCII in place of JIS0201-1976-Roman?
3994 9. Use JISX0208-1983 in place of JISX0208-1978?
3995 These specifications are encoded in CODING_ISO_FLAGS (coding) as flag bits
3996 defined by macros CODING_ISO_FLAG_XXX. See `coding.h' for more
4000 /* Produce codes (escape sequence) for designating CHARSET to graphic
4001 register REG at DST, and increment DST. If <final-char> of CHARSET is
4002 '@', 'A', or 'B' and the coding system CODING allows, produce
4003 designation sequence of short-form. */
4005 #define ENCODE_DESIGNATION(charset, reg, coding) \
4007 unsigned char final_char = CHARSET_ISO_FINAL (charset); \
4008 const char *intermediate_char_94 = "()*+"; \
4009 const char *intermediate_char_96 = ",-./"; \
4010 int revision = -1; \
4012 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_REVISION) \
4013 revision = CHARSET_ISO_REVISION (charset); \
4015 if (revision >= 0) \
4017 EMIT_TWO_ASCII_BYTES (ISO_CODE_ESC, '&'); \
4018 EMIT_ONE_BYTE ('@' + revision); \
4020 EMIT_ONE_ASCII_BYTE (ISO_CODE_ESC); \
4021 if (CHARSET_DIMENSION (charset) == 1) \
4024 if (! CHARSET_ISO_CHARS_96 (charset)) \
4025 b = intermediate_char_94[reg]; \
4027 b = intermediate_char_96[reg]; \
4028 EMIT_ONE_ASCII_BYTE (b); \
4032 EMIT_ONE_ASCII_BYTE ('$'); \
4033 if (! CHARSET_ISO_CHARS_96 (charset)) \
4035 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_LONG_FORM \
4037 || final_char < '@' || final_char > 'B') \
4038 EMIT_ONE_ASCII_BYTE (intermediate_char_94[reg]); \
4041 EMIT_ONE_ASCII_BYTE (intermediate_char_96[reg]); \
4043 EMIT_ONE_ASCII_BYTE (final_char); \
4045 CODING_ISO_DESIGNATION (coding, reg) = CHARSET_ID (charset); \
4049 /* The following two macros produce codes (control character or escape
4050 sequence) for ISO2022 single-shift functions (single-shift-2 and
4053 #define ENCODE_SINGLE_SHIFT_2 \
4055 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_SEVEN_BITS) \
4056 EMIT_TWO_ASCII_BYTES (ISO_CODE_ESC, 'N'); \
4058 EMIT_ONE_BYTE (ISO_CODE_SS2); \
4059 CODING_ISO_SINGLE_SHIFTING (coding) = 1; \
4063 #define ENCODE_SINGLE_SHIFT_3 \
4065 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_SEVEN_BITS) \
4066 EMIT_TWO_ASCII_BYTES (ISO_CODE_ESC, 'O'); \
4068 EMIT_ONE_BYTE (ISO_CODE_SS3); \
4069 CODING_ISO_SINGLE_SHIFTING (coding) = 1; \
4073 /* The following four macros produce codes (control character or
4074 escape sequence) for ISO2022 locking-shift functions (shift-in,
4075 shift-out, locking-shift-2, and locking-shift-3). */
4077 #define ENCODE_SHIFT_IN \
4079 EMIT_ONE_ASCII_BYTE (ISO_CODE_SI); \
4080 CODING_ISO_INVOCATION (coding, 0) = 0; \
4084 #define ENCODE_SHIFT_OUT \
4086 EMIT_ONE_ASCII_BYTE (ISO_CODE_SO); \
4087 CODING_ISO_INVOCATION (coding, 0) = 1; \
4091 #define ENCODE_LOCKING_SHIFT_2 \
4093 EMIT_TWO_ASCII_BYTES (ISO_CODE_ESC, 'n'); \
4094 CODING_ISO_INVOCATION (coding, 0) = 2; \
4098 #define ENCODE_LOCKING_SHIFT_3 \
4100 EMIT_TWO_ASCII_BYTES (ISO_CODE_ESC, 'n'); \
4101 CODING_ISO_INVOCATION (coding, 0) = 3; \
4105 /* Produce codes for a DIMENSION1 character whose character set is
4106 CHARSET and whose position-code is C1. Designation and invocation
4107 sequences are also produced in advance if necessary. */
4109 #define ENCODE_ISO_CHARACTER_DIMENSION1(charset, c1) \
4111 int id = CHARSET_ID (charset); \
4113 if ((CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_USE_ROMAN) \
4114 && id == charset_ascii) \
4116 id = charset_jisx0201_roman; \
4117 charset = CHARSET_FROM_ID (id); \
4120 if (CODING_ISO_SINGLE_SHIFTING (coding)) \
4122 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_SEVEN_BITS) \
4123 EMIT_ONE_ASCII_BYTE (c1 & 0x7F); \
4125 EMIT_ONE_BYTE (c1 | 0x80); \
4126 CODING_ISO_SINGLE_SHIFTING (coding) = 0; \
4129 else if (id == CODING_ISO_INVOKED_CHARSET (coding, 0)) \
4131 EMIT_ONE_ASCII_BYTE (c1 & 0x7F); \
4134 else if (id == CODING_ISO_INVOKED_CHARSET (coding, 1)) \
4136 EMIT_ONE_BYTE (c1 | 0x80); \
4140 /* Since CHARSET is not yet invoked to any graphic planes, we \
4141 must invoke it, or, at first, designate it to some graphic \
4142 register. Then repeat the loop to actually produce the \
4144 dst = encode_invocation_designation (charset, coding, dst, \
4149 /* Produce codes for a DIMENSION2 character whose character set is
4150 CHARSET and whose position-codes are C1 and C2. Designation and
4151 invocation codes are also produced in advance if necessary. */
4153 #define ENCODE_ISO_CHARACTER_DIMENSION2(charset, c1, c2) \
4155 int id = CHARSET_ID (charset); \
4157 if ((CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_USE_OLDJIS) \
4158 && id == charset_jisx0208) \
4160 id = charset_jisx0208_1978; \
4161 charset = CHARSET_FROM_ID (id); \
4164 if (CODING_ISO_SINGLE_SHIFTING (coding)) \
4166 if (CODING_ISO_FLAGS (coding) & CODING_ISO_FLAG_SEVEN_BITS) \
4167 EMIT_TWO_ASCII_BYTES ((c1) & 0x7F, (c2) & 0x7F); \
4169 EMIT_TWO_BYTES ((c1) | 0x80, (c2) | 0x80); \
4170 CODING_ISO_SINGLE_SHIFTING (coding) = 0; \
4173 else if (id == CODING_ISO_INVOKED_CHARSET (coding, 0)) \
4175 EMIT_TWO_ASCII_BYTES ((c1) & 0x7F, (c2) & 0x7F); \
4178 else if (id == CODING_ISO_INVOKED_CHARSET (coding, 1)) \
4180 EMIT_TWO_BYTES ((c1) | 0x80, (c2) | 0x80); \
4184 /* Since CHARSET is not yet invoked to any graphic planes, we \
4185 must invoke it, or, at first, designate it to some graphic \
4186 register. Then repeat the loop to actually produce the \
4188 dst = encode_invocation_designation (charset, coding, dst, \
4193 #define ENCODE_ISO_CHARACTER(charset, c) \
4196 CODING_ENCODE_CHAR (coding, dst, dst_end, (charset), (c), code); \
4198 if (CHARSET_DIMENSION (charset) == 1) \
4199 ENCODE_ISO_CHARACTER_DIMENSION1 ((charset), code); \
4201 ENCODE_ISO_CHARACTER_DIMENSION2 ((charset), code >> 8, code & 0xFF); \
4205 /* Produce designation and invocation codes at a place pointed by DST
4206 to use CHARSET. The element `spec.iso_2022' of *CODING is updated.
4209 static unsigned char *
4210 encode_invocation_designation (struct charset
*charset
,
4211 struct coding_system
*coding
,
4212 unsigned char *dst
, ptrdiff_t *p_nchars
)
4214 bool multibytep
= coding
->dst_multibyte
;
4215 ptrdiff_t produced_chars
= *p_nchars
;
4216 int reg
; /* graphic register number */
4217 int id
= CHARSET_ID (charset
);
4219 /* At first, check designations. */
4220 for (reg
= 0; reg
< 4; reg
++)
4221 if (id
== CODING_ISO_DESIGNATION (coding
, reg
))
4226 /* CHARSET is not yet designated to any graphic registers. */
4227 /* At first check the requested designation. */
4228 reg
= CODING_ISO_REQUEST (coding
, id
);
4230 /* Since CHARSET requests no special designation, designate it
4231 to graphic register 0. */
4234 ENCODE_DESIGNATION (charset
, reg
, coding
);
4237 if (CODING_ISO_INVOCATION (coding
, 0) != reg
4238 && CODING_ISO_INVOCATION (coding
, 1) != reg
)
4240 /* Since the graphic register REG is not invoked to any graphic
4241 planes, invoke it to graphic plane 0. */
4244 case 0: /* graphic register 0 */
4248 case 1: /* graphic register 1 */
4252 case 2: /* graphic register 2 */
4253 if (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SINGLE_SHIFT
)
4254 ENCODE_SINGLE_SHIFT_2
;
4256 ENCODE_LOCKING_SHIFT_2
;
4259 case 3: /* graphic register 3 */
4260 if (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_SINGLE_SHIFT
)
4261 ENCODE_SINGLE_SHIFT_3
;
4263 ENCODE_LOCKING_SHIFT_3
;
4268 *p_nchars
= produced_chars
;
4273 /* Produce codes for designation and invocation to reset the graphic
4274 planes and registers to initial state. */
4275 #define ENCODE_RESET_PLANE_AND_REGISTER() \
4278 struct charset *charset; \
4280 if (CODING_ISO_INVOCATION (coding, 0) != 0) \
4282 for (reg = 0; reg < 4; reg++) \
4283 if (CODING_ISO_INITIAL (coding, reg) >= 0 \
4284 && (CODING_ISO_DESIGNATION (coding, reg) \
4285 != CODING_ISO_INITIAL (coding, reg))) \
4287 charset = CHARSET_FROM_ID (CODING_ISO_INITIAL (coding, reg)); \
4288 ENCODE_DESIGNATION (charset, reg, coding); \
4293 /* Produce designation sequences of charsets in the line started from
4294 CHARBUF to a place pointed by DST, and return the number of
4295 produced bytes. DST should not directly point a buffer text area
4296 which may be relocated by char_charset call.
4298 If the current block ends before any end-of-line, we may fail to
4299 find all the necessary designations. */
4302 encode_designation_at_bol (struct coding_system
*coding
,
4303 int *charbuf
, int *charbuf_end
,
4306 unsigned char *orig
= dst
;
4307 struct charset
*charset
;
4308 /* Table of charsets to be designated to each graphic register. */
4310 int c
, found
= 0, reg
;
4311 ptrdiff_t produced_chars
= 0;
4312 bool multibytep
= coding
->dst_multibyte
;
4314 Lisp_Object charset_list
;
4316 attrs
= CODING_ID_ATTRS (coding
->id
);
4317 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
4318 if (EQ (charset_list
, Qiso_2022
))
4319 charset_list
= Viso_2022_charset_list
;
4321 for (reg
= 0; reg
< 4; reg
++)
4324 while (charbuf
< charbuf_end
&& found
< 4)
4331 charset
= char_charset (c
, charset_list
, NULL
);
4332 id
= CHARSET_ID (charset
);
4333 reg
= CODING_ISO_REQUEST (coding
, id
);
4334 if (reg
>= 0 && r
[reg
] < 0)
4343 for (reg
= 0; reg
< 4; reg
++)
4345 && CODING_ISO_DESIGNATION (coding
, reg
) != r
[reg
])
4346 ENCODE_DESIGNATION (CHARSET_FROM_ID (r
[reg
]), reg
, coding
);
4352 /* See the above "GENERAL NOTES on `encode_coding_XXX ()' functions". */
4355 encode_coding_iso_2022 (struct coding_system
*coding
)
4357 bool multibytep
= coding
->dst_multibyte
;
4358 int *charbuf
= coding
->charbuf
;
4359 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
4360 unsigned char *dst
= coding
->destination
+ coding
->produced
;
4361 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
4363 bool bol_designation
4364 = (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_DESIGNATE_AT_BOL
4365 && CODING_ISO_BOL (coding
));
4366 ptrdiff_t produced_chars
= 0;
4367 Lisp_Object attrs
, eol_type
, charset_list
;
4368 bool ascii_compatible
;
4370 int preferred_charset_id
= -1;
4372 CODING_GET_INFO (coding
, attrs
, charset_list
);
4373 eol_type
= inhibit_eol_conversion
? Qunix
: CODING_ID_EOL_TYPE (coding
->id
);
4374 if (VECTORP (eol_type
))
4377 setup_iso_safe_charsets (attrs
);
4378 /* Charset list may have been changed. */
4379 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
4380 coding
->safe_charsets
= SDATA (CODING_ATTR_SAFE_CHARSETS (attrs
));
4383 = (! NILP (CODING_ATTR_ASCII_COMPAT (attrs
))
4384 && ! (CODING_ISO_FLAGS (coding
) & (CODING_ISO_FLAG_DESIGNATION
4385 | CODING_ISO_FLAG_LOCKING_SHIFT
)));
4387 while (charbuf
< charbuf_end
)
4389 ASSURE_DESTINATION (safe_room
);
4391 if (bol_designation
)
4393 /* We have to produce designation sequences if any now. */
4394 unsigned char desig_buf
[16];
4398 charset_map_loaded
= 0;
4399 nbytes
= encode_designation_at_bol (coding
, charbuf
, charbuf_end
,
4401 if (charset_map_loaded
4402 && (offset
= coding_change_destination (coding
)))
4407 memcpy (dst
, desig_buf
, nbytes
);
4409 /* We are sure that designation sequences are all ASCII bytes. */
4410 produced_chars
+= nbytes
;
4411 bol_designation
= 0;
4412 ASSURE_DESTINATION (safe_room
);
4419 /* Handle an annotation. */
4422 case CODING_ANNOTATE_COMPOSITION_MASK
:
4423 /* Not yet implemented. */
4425 case CODING_ANNOTATE_CHARSET_MASK
:
4426 preferred_charset_id
= charbuf
[2];
4427 if (preferred_charset_id
>= 0
4428 && NILP (Fmemq (make_number (preferred_charset_id
),
4430 preferred_charset_id
= -1;
4439 /* Now encode the character C. */
4440 if (c
< 0x20 || c
== 0x7F)
4443 || (c
== '\r' && EQ (eol_type
, Qmac
)))
4445 if (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_RESET_AT_EOL
)
4446 ENCODE_RESET_PLANE_AND_REGISTER ();
4447 if (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_INIT_AT_BOL
)
4451 for (i
= 0; i
< 4; i
++)
4452 CODING_ISO_DESIGNATION (coding
, i
)
4453 = CODING_ISO_INITIAL (coding
, i
);
4455 bol_designation
= ((CODING_ISO_FLAGS (coding
)
4456 & CODING_ISO_FLAG_DESIGNATE_AT_BOL
)
4459 else if (CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_RESET_AT_CNTL
)
4460 ENCODE_RESET_PLANE_AND_REGISTER ();
4461 EMIT_ONE_ASCII_BYTE (c
);
4463 else if (ASCII_CHAR_P (c
))
4465 if (ascii_compatible
)
4466 EMIT_ONE_ASCII_BYTE (c
);
4469 struct charset
*charset
= CHARSET_FROM_ID (charset_ascii
);
4470 ENCODE_ISO_CHARACTER (charset
, c
);
4473 else if (CHAR_BYTE8_P (c
))
4475 c
= CHAR_TO_BYTE8 (c
);
4480 struct charset
*charset
;
4482 if (preferred_charset_id
>= 0)
4486 charset
= CHARSET_FROM_ID (preferred_charset_id
);
4487 CODING_CHAR_CHARSET_P (coding
, dst
, dst_end
, c
, charset
, result
);
4489 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
4493 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
4497 if (coding
->mode
& CODING_MODE_SAFE_ENCODING
)
4499 c
= CODING_INHIBIT_CHARACTER_SUBSTITUTION
;
4500 charset
= CHARSET_FROM_ID (charset_ascii
);
4504 c
= coding
->default_char
;
4505 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
,
4506 charset_list
, NULL
, charset
);
4509 ENCODE_ISO_CHARACTER (charset
, c
);
4513 if (coding
->mode
& CODING_MODE_LAST_BLOCK
4514 && CODING_ISO_FLAGS (coding
) & CODING_ISO_FLAG_RESET_AT_EOL
)
4516 ASSURE_DESTINATION (safe_room
);
4517 ENCODE_RESET_PLANE_AND_REGISTER ();
4519 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
4520 CODING_ISO_BOL (coding
) = bol_designation
;
4521 coding
->produced_char
+= produced_chars
;
4522 coding
->produced
= dst
- coding
->destination
;
4527 /*** 8,9. SJIS and BIG5 handlers ***/
4529 /* Although SJIS and BIG5 are not ISO's coding system, they are used
4530 quite widely. So, for the moment, Emacs supports them in the bare
4531 C code. But, in the future, they may be supported only by CCL. */
4533 /* SJIS is a coding system encoding three character sets: ASCII, right
4534 half of JISX0201-Kana, and JISX0208. An ASCII character is encoded
4535 as is. A character of charset katakana-jisx0201 is encoded by
4536 "position-code + 0x80". A character of charset japanese-jisx0208
4537 is encoded in 2-byte but two position-codes are divided and shifted
4538 so that it fit in the range below.
4540 --- CODE RANGE of SJIS ---
4541 (character set) (range)
4543 KATAKANA-JISX0201 0xA0 .. 0xDF
4544 JISX0208 (1st byte) 0x81 .. 0x9F and 0xE0 .. 0xEF
4545 (2nd byte) 0x40 .. 0x7E and 0x80 .. 0xFC
4546 -------------------------------
4550 /* BIG5 is a coding system encoding two character sets: ASCII and
4551 Big5. An ASCII character is encoded as is. Big5 is a two-byte
4552 character set and is encoded in two-byte.
4554 --- CODE RANGE of BIG5 ---
4555 (character set) (range)
4557 Big5 (1st byte) 0xA1 .. 0xFE
4558 (2nd byte) 0x40 .. 0x7E and 0xA1 .. 0xFE
4559 --------------------------
4563 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
4564 Return true if a text is encoded in SJIS. */
4567 detect_coding_sjis (struct coding_system
*coding
,
4568 struct coding_detection_info
*detect_info
)
4570 const unsigned char *src
= coding
->source
, *src_base
;
4571 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
4572 bool multibytep
= coding
->src_multibyte
;
4573 ptrdiff_t consumed_chars
= 0;
4576 Lisp_Object attrs
, charset_list
;
4577 int max_first_byte_of_2_byte_code
;
4579 CODING_GET_INFO (coding
, attrs
, charset_list
);
4580 max_first_byte_of_2_byte_code
4581 = (XINT (Flength (charset_list
)) > 3 ? 0xFC : 0xEF);
4583 detect_info
->checked
|= CATEGORY_MASK_SJIS
;
4584 /* A coding system of this category is always ASCII compatible. */
4585 src
+= coding
->head_ascii
;
4593 if ((c
>= 0x81 && c
<= 0x9F)
4594 || (c
>= 0xE0 && c
<= max_first_byte_of_2_byte_code
))
4597 if (c
< 0x40 || c
== 0x7F || c
> 0xFC)
4599 found
= CATEGORY_MASK_SJIS
;
4601 else if (c
>= 0xA0 && c
< 0xE0)
4602 found
= CATEGORY_MASK_SJIS
;
4606 detect_info
->rejected
|= CATEGORY_MASK_SJIS
;
4610 if (src_base
< src
&& coding
->mode
& CODING_MODE_LAST_BLOCK
)
4612 detect_info
->rejected
|= CATEGORY_MASK_SJIS
;
4615 detect_info
->found
|= found
;
4619 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
4620 Return true if a text is encoded in BIG5. */
4623 detect_coding_big5 (struct coding_system
*coding
,
4624 struct coding_detection_info
*detect_info
)
4626 const unsigned char *src
= coding
->source
, *src_base
;
4627 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
4628 bool multibytep
= coding
->src_multibyte
;
4629 ptrdiff_t consumed_chars
= 0;
4633 detect_info
->checked
|= CATEGORY_MASK_BIG5
;
4634 /* A coding system of this category is always ASCII compatible. */
4635 src
+= coding
->head_ascii
;
4646 if (c
< 0x40 || (c
>= 0x7F && c
<= 0xA0))
4648 found
= CATEGORY_MASK_BIG5
;
4653 detect_info
->rejected
|= CATEGORY_MASK_BIG5
;
4657 if (src_base
< src
&& coding
->mode
& CODING_MODE_LAST_BLOCK
)
4659 detect_info
->rejected
|= CATEGORY_MASK_BIG5
;
4662 detect_info
->found
|= found
;
4666 /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions". */
4669 decode_coding_sjis (struct coding_system
*coding
)
4671 const unsigned char *src
= coding
->source
+ coding
->consumed
;
4672 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
4673 const unsigned char *src_base
;
4674 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
4675 /* We may produce one charset annotation in one loop and one more at
4678 = coding
->charbuf
+ coding
->charbuf_size
- (MAX_ANNOTATION_LENGTH
* 2);
4679 ptrdiff_t consumed_chars
= 0, consumed_chars_base
;
4680 bool multibytep
= coding
->src_multibyte
;
4681 struct charset
*charset_roman
, *charset_kanji
, *charset_kana
;
4682 struct charset
*charset_kanji2
;
4683 Lisp_Object attrs
, charset_list
, val
;
4684 ptrdiff_t char_offset
= coding
->produced_char
;
4685 ptrdiff_t last_offset
= char_offset
;
4686 int last_id
= charset_ascii
;
4688 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
4689 int byte_after_cr
= -1;
4691 CODING_GET_INFO (coding
, attrs
, charset_list
);
4694 charset_roman
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
4695 charset_kana
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
4696 charset_kanji
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
4697 charset_kanji2
= NILP (val
) ? NULL
: CHARSET_FROM_ID (XINT (XCAR (val
)));
4702 struct charset
*charset
;
4705 consumed_chars_base
= consumed_chars
;
4707 if (charbuf
>= charbuf_end
)
4709 if (byte_after_cr
>= 0)
4714 if (byte_after_cr
>= 0)
4715 c
= byte_after_cr
, byte_after_cr
= -1;
4722 if (eol_dos
&& c
== '\r')
4723 ONE_MORE_BYTE (byte_after_cr
);
4724 charset
= charset_roman
;
4726 else if (c
== 0x80 || c
== 0xA0)
4728 else if (c
>= 0xA1 && c
<= 0xDF)
4730 /* SJIS -> JISX0201-Kana */
4732 charset
= charset_kana
;
4736 /* SJIS -> JISX0208 */
4738 if (c1
< 0x40 || c1
== 0x7F || c1
> 0xFC)
4742 charset
= charset_kanji
;
4744 else if (c
<= 0xFC && charset_kanji2
)
4746 /* SJIS -> JISX0213-2 */
4748 if (c1
< 0x40 || c1
== 0x7F || c1
> 0xFC)
4752 charset
= charset_kanji2
;
4756 if (charset
->id
!= charset_ascii
4757 && last_id
!= charset
->id
)
4759 if (last_id
!= charset_ascii
)
4760 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
4761 last_id
= charset
->id
;
4762 last_offset
= char_offset
;
4764 CODING_DECODE_CHAR (coding
, src
, src_base
, src_end
, charset
, c
, c
);
4771 consumed_chars
= consumed_chars_base
;
4773 *charbuf
++ = c
< 0 ? -c
: BYTE8_TO_CHAR (c
);
4779 if (last_id
!= charset_ascii
)
4780 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
4781 coding
->consumed_char
+= consumed_chars_base
;
4782 coding
->consumed
= src_base
- coding
->source
;
4783 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
4787 decode_coding_big5 (struct coding_system
*coding
)
4789 const unsigned char *src
= coding
->source
+ coding
->consumed
;
4790 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
4791 const unsigned char *src_base
;
4792 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
4793 /* We may produce one charset annotation in one loop and one more at
4796 = coding
->charbuf
+ coding
->charbuf_size
- (MAX_ANNOTATION_LENGTH
* 2);
4797 ptrdiff_t consumed_chars
= 0, consumed_chars_base
;
4798 bool multibytep
= coding
->src_multibyte
;
4799 struct charset
*charset_roman
, *charset_big5
;
4800 Lisp_Object attrs
, charset_list
, val
;
4801 ptrdiff_t char_offset
= coding
->produced_char
;
4802 ptrdiff_t last_offset
= char_offset
;
4803 int last_id
= charset_ascii
;
4805 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
4806 int byte_after_cr
= -1;
4808 CODING_GET_INFO (coding
, attrs
, charset_list
);
4810 charset_roman
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
4811 charset_big5
= CHARSET_FROM_ID (XINT (XCAR (val
)));
4816 struct charset
*charset
;
4819 consumed_chars_base
= consumed_chars
;
4821 if (charbuf
>= charbuf_end
)
4823 if (byte_after_cr
>= 0)
4828 if (byte_after_cr
>= 0)
4829 c
= byte_after_cr
, byte_after_cr
= -1;
4837 if (eol_dos
&& c
== '\r')
4838 ONE_MORE_BYTE (byte_after_cr
);
4839 charset
= charset_roman
;
4844 if (c
< 0xA1 || c
> 0xFE)
4847 if (c1
< 0x40 || (c1
> 0x7E && c1
< 0xA1) || c1
> 0xFE)
4850 charset
= charset_big5
;
4852 if (charset
->id
!= charset_ascii
4853 && last_id
!= charset
->id
)
4855 if (last_id
!= charset_ascii
)
4856 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
4857 last_id
= charset
->id
;
4858 last_offset
= char_offset
;
4860 CODING_DECODE_CHAR (coding
, src
, src_base
, src_end
, charset
, c
, c
);
4867 consumed_chars
= consumed_chars_base
;
4869 *charbuf
++ = c
< 0 ? -c
: BYTE8_TO_CHAR (c
);
4875 if (last_id
!= charset_ascii
)
4876 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
4877 coding
->consumed_char
+= consumed_chars_base
;
4878 coding
->consumed
= src_base
- coding
->source
;
4879 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
4882 /* See the above "GENERAL NOTES on `encode_coding_XXX ()' functions".
4883 This function can encode charsets `ascii', `katakana-jisx0201',
4884 `japanese-jisx0208', `chinese-big5-1', and `chinese-big5-2'. We
4885 are sure that all these charsets are registered as official charset
4886 (i.e. do not have extended leading-codes). Characters of other
4887 charsets are produced without any encoding. */
4890 encode_coding_sjis (struct coding_system
*coding
)
4892 bool multibytep
= coding
->dst_multibyte
;
4893 int *charbuf
= coding
->charbuf
;
4894 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
4895 unsigned char *dst
= coding
->destination
+ coding
->produced
;
4896 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
4898 ptrdiff_t produced_chars
= 0;
4899 Lisp_Object attrs
, charset_list
, val
;
4900 bool ascii_compatible
;
4901 struct charset
*charset_kanji
, *charset_kana
;
4902 struct charset
*charset_kanji2
;
4905 CODING_GET_INFO (coding
, attrs
, charset_list
);
4906 val
= XCDR (charset_list
);
4907 charset_kana
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
4908 charset_kanji
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
4909 charset_kanji2
= NILP (val
) ? NULL
: CHARSET_FROM_ID (XINT (XCAR (val
)));
4911 ascii_compatible
= ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
));
4913 while (charbuf
< charbuf_end
)
4915 ASSURE_DESTINATION (safe_room
);
4917 /* Now encode the character C. */
4918 if (ASCII_CHAR_P (c
) && ascii_compatible
)
4919 EMIT_ONE_ASCII_BYTE (c
);
4920 else if (CHAR_BYTE8_P (c
))
4922 c
= CHAR_TO_BYTE8 (c
);
4928 struct charset
*charset
;
4929 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
4934 if (coding
->mode
& CODING_MODE_SAFE_ENCODING
)
4936 code
= CODING_INHIBIT_CHARACTER_SUBSTITUTION
;
4937 charset
= CHARSET_FROM_ID (charset_ascii
);
4941 c
= coding
->default_char
;
4942 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
,
4943 charset_list
, &code
, charset
);
4946 if (code
== CHARSET_INVALID_CODE (charset
))
4948 if (charset
== charset_kanji
)
4952 c1
= code
>> 8, c2
= code
& 0xFF;
4953 EMIT_TWO_BYTES (c1
, c2
);
4955 else if (charset
== charset_kana
)
4956 EMIT_ONE_BYTE (code
| 0x80);
4957 else if (charset_kanji2
&& charset
== charset_kanji2
)
4962 if (c1
== 0x21 || (c1
>= 0x23 && c1
<= 0x25)
4964 || (c1
>= 0x2C && c1
<= 0x2F) || c1
>= 0x6E)
4966 JIS_TO_SJIS2 (code
);
4967 c1
= code
>> 8, c2
= code
& 0xFF;
4968 EMIT_TWO_BYTES (c1
, c2
);
4971 EMIT_ONE_ASCII_BYTE (code
& 0x7F);
4974 EMIT_ONE_ASCII_BYTE (code
& 0x7F);
4977 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
4978 coding
->produced_char
+= produced_chars
;
4979 coding
->produced
= dst
- coding
->destination
;
4984 encode_coding_big5 (struct coding_system
*coding
)
4986 bool multibytep
= coding
->dst_multibyte
;
4987 int *charbuf
= coding
->charbuf
;
4988 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
4989 unsigned char *dst
= coding
->destination
+ coding
->produced
;
4990 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
4992 ptrdiff_t produced_chars
= 0;
4993 Lisp_Object attrs
, charset_list
, val
;
4994 bool ascii_compatible
;
4995 struct charset
*charset_big5
;
4998 CODING_GET_INFO (coding
, attrs
, charset_list
);
4999 val
= XCDR (charset_list
);
5000 charset_big5
= CHARSET_FROM_ID (XINT (XCAR (val
)));
5001 ascii_compatible
= ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
));
5003 while (charbuf
< charbuf_end
)
5005 ASSURE_DESTINATION (safe_room
);
5007 /* Now encode the character C. */
5008 if (ASCII_CHAR_P (c
) && ascii_compatible
)
5009 EMIT_ONE_ASCII_BYTE (c
);
5010 else if (CHAR_BYTE8_P (c
))
5012 c
= CHAR_TO_BYTE8 (c
);
5018 struct charset
*charset
;
5019 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
5024 if (coding
->mode
& CODING_MODE_SAFE_ENCODING
)
5026 code
= CODING_INHIBIT_CHARACTER_SUBSTITUTION
;
5027 charset
= CHARSET_FROM_ID (charset_ascii
);
5031 c
= coding
->default_char
;
5032 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
,
5033 charset_list
, &code
, charset
);
5036 if (code
== CHARSET_INVALID_CODE (charset
))
5038 if (charset
== charset_big5
)
5042 c1
= code
>> 8, c2
= code
& 0xFF;
5043 EMIT_TWO_BYTES (c1
, c2
);
5046 EMIT_ONE_ASCII_BYTE (code
& 0x7F);
5049 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
5050 coding
->produced_char
+= produced_chars
;
5051 coding
->produced
= dst
- coding
->destination
;
5056 /*** 10. CCL handlers ***/
5058 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
5059 Return true if a text is encoded in a coding system of which
5060 encoder/decoder are written in CCL program. */
5063 detect_coding_ccl (struct coding_system
*coding
,
5064 struct coding_detection_info
*detect_info
)
5066 const unsigned char *src
= coding
->source
, *src_base
;
5067 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
5068 bool multibytep
= coding
->src_multibyte
;
5069 ptrdiff_t consumed_chars
= 0;
5071 unsigned char *valids
;
5072 ptrdiff_t head_ascii
= coding
->head_ascii
;
5075 detect_info
->checked
|= CATEGORY_MASK_CCL
;
5077 coding
= &coding_categories
[coding_category_ccl
];
5078 valids
= CODING_CCL_VALIDS (coding
);
5079 attrs
= CODING_ID_ATTRS (coding
->id
);
5080 if (! NILP (CODING_ATTR_ASCII_COMPAT (attrs
)))
5089 if (c
< 0 || ! valids
[c
])
5091 if ((valids
[c
] > 1))
5092 found
= CATEGORY_MASK_CCL
;
5094 detect_info
->rejected
|= CATEGORY_MASK_CCL
;
5098 detect_info
->found
|= found
;
5103 decode_coding_ccl (struct coding_system
*coding
)
5105 const unsigned char *src
= coding
->source
+ coding
->consumed
;
5106 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
5107 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
5108 int *charbuf_end
= coding
->charbuf
+ coding
->charbuf_size
;
5109 ptrdiff_t consumed_chars
= 0;
5110 bool multibytep
= coding
->src_multibyte
;
5111 struct ccl_program
*ccl
= &coding
->spec
.ccl
->ccl
;
5112 int source_charbuf
[1024];
5113 int source_byteidx
[1025];
5114 Lisp_Object attrs
, charset_list
;
5116 CODING_GET_INFO (coding
, attrs
, charset_list
);
5120 const unsigned char *p
= src
;
5126 while (i
< 1024 && p
< src_end
)
5128 source_byteidx
[i
] = p
- src
;
5129 source_charbuf
[i
++] = STRING_CHAR_ADVANCE (p
);
5131 source_byteidx
[i
] = p
- src
;
5134 while (i
< 1024 && p
< src_end
)
5135 source_charbuf
[i
++] = *p
++;
5137 if (p
== src_end
&& coding
->mode
& CODING_MODE_LAST_BLOCK
)
5138 ccl
->last_block
= 1;
5139 /* As ccl_driver calls DECODE_CHAR, buffer may be relocated. */
5140 charset_map_loaded
= 0;
5141 ccl_driver (ccl
, source_charbuf
, charbuf
, i
, charbuf_end
- charbuf
,
5143 if (charset_map_loaded
5144 && (offset
= coding_change_source (coding
)))
5150 charbuf
+= ccl
->produced
;
5152 src
+= source_byteidx
[ccl
->consumed
];
5154 src
+= ccl
->consumed
;
5155 consumed_chars
+= ccl
->consumed
;
5156 if (p
== src_end
|| ccl
->status
!= CCL_STAT_SUSPEND_BY_SRC
)
5160 switch (ccl
->status
)
5162 case CCL_STAT_SUSPEND_BY_SRC
:
5163 record_conversion_result (coding
, CODING_RESULT_INSUFFICIENT_SRC
);
5165 case CCL_STAT_SUSPEND_BY_DST
:
5166 record_conversion_result (coding
, CODING_RESULT_INSUFFICIENT_DST
);
5169 case CCL_STAT_INVALID_CMD
:
5170 record_conversion_result (coding
, CODING_RESULT_INTERRUPT
);
5173 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
5176 coding
->consumed_char
+= consumed_chars
;
5177 coding
->consumed
= src
- coding
->source
;
5178 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
5182 encode_coding_ccl (struct coding_system
*coding
)
5184 struct ccl_program
*ccl
= &coding
->spec
.ccl
->ccl
;
5185 bool multibytep
= coding
->dst_multibyte
;
5186 int *charbuf
= coding
->charbuf
;
5187 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
5188 unsigned char *dst
= coding
->destination
+ coding
->produced
;
5189 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
5190 int destination_charbuf
[1024];
5191 ptrdiff_t produced_chars
= 0;
5193 Lisp_Object attrs
, charset_list
;
5195 CODING_GET_INFO (coding
, attrs
, charset_list
);
5196 if (coding
->consumed_char
== coding
->src_chars
5197 && coding
->mode
& CODING_MODE_LAST_BLOCK
)
5198 ccl
->last_block
= 1;
5204 /* As ccl_driver calls DECODE_CHAR, buffer may be relocated. */
5205 charset_map_loaded
= 0;
5206 ccl_driver (ccl
, charbuf
, destination_charbuf
,
5207 charbuf_end
- charbuf
, 1024, charset_list
);
5208 if (charset_map_loaded
5209 && (offset
= coding_change_destination (coding
)))
5213 ASSURE_DESTINATION (ccl
->produced
* 2);
5214 for (i
= 0; i
< ccl
->produced
; i
++)
5215 EMIT_ONE_BYTE (destination_charbuf
[i
] & 0xFF);
5219 ASSURE_DESTINATION (ccl
->produced
);
5220 for (i
= 0; i
< ccl
->produced
; i
++)
5221 *dst
++ = destination_charbuf
[i
] & 0xFF;
5222 produced_chars
+= ccl
->produced
;
5224 charbuf
+= ccl
->consumed
;
5225 if (ccl
->status
== CCL_STAT_QUIT
5226 || ccl
->status
== CCL_STAT_INVALID_CMD
)
5229 while (charbuf
< charbuf_end
);
5231 switch (ccl
->status
)
5233 case CCL_STAT_SUSPEND_BY_SRC
:
5234 record_conversion_result (coding
, CODING_RESULT_INSUFFICIENT_SRC
);
5236 case CCL_STAT_SUSPEND_BY_DST
:
5237 record_conversion_result (coding
, CODING_RESULT_INSUFFICIENT_DST
);
5240 case CCL_STAT_INVALID_CMD
:
5241 record_conversion_result (coding
, CODING_RESULT_INTERRUPT
);
5244 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
5248 coding
->produced_char
+= produced_chars
;
5249 coding
->produced
= dst
- coding
->destination
;
5254 /*** 10, 11. no-conversion handlers ***/
5256 /* See the above "GENERAL NOTES on `decode_coding_XXX ()' functions". */
5259 decode_coding_raw_text (struct coding_system
*coding
)
5262 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
5264 coding
->chars_at_source
= 1;
5265 coding
->consumed_char
= coding
->src_chars
;
5266 coding
->consumed
= coding
->src_bytes
;
5267 if (eol_dos
&& coding
->source
[coding
->src_bytes
- 1] == '\r')
5269 coding
->consumed_char
--;
5271 record_conversion_result (coding
, CODING_RESULT_INSUFFICIENT_SRC
);
5274 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
5278 encode_coding_raw_text (struct coding_system
*coding
)
5280 bool multibytep
= coding
->dst_multibyte
;
5281 int *charbuf
= coding
->charbuf
;
5282 int *charbuf_end
= coding
->charbuf
+ coding
->charbuf_used
;
5283 unsigned char *dst
= coding
->destination
+ coding
->produced
;
5284 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
5285 ptrdiff_t produced_chars
= 0;
5290 int safe_room
= MAX_MULTIBYTE_LENGTH
* 2;
5292 if (coding
->src_multibyte
)
5293 while (charbuf
< charbuf_end
)
5295 ASSURE_DESTINATION (safe_room
);
5297 if (ASCII_CHAR_P (c
))
5298 EMIT_ONE_ASCII_BYTE (c
);
5299 else if (CHAR_BYTE8_P (c
))
5301 c
= CHAR_TO_BYTE8 (c
);
5306 unsigned char str
[MAX_MULTIBYTE_LENGTH
], *p0
= str
, *p1
= str
;
5308 CHAR_STRING_ADVANCE (c
, p1
);
5311 EMIT_ONE_BYTE (*p0
);
5318 while (charbuf
< charbuf_end
)
5320 ASSURE_DESTINATION (safe_room
);
5327 if (coding
->src_multibyte
)
5329 int safe_room
= MAX_MULTIBYTE_LENGTH
;
5331 while (charbuf
< charbuf_end
)
5333 ASSURE_DESTINATION (safe_room
);
5335 if (ASCII_CHAR_P (c
))
5337 else if (CHAR_BYTE8_P (c
))
5338 *dst
++ = CHAR_TO_BYTE8 (c
);
5340 CHAR_STRING_ADVANCE (c
, dst
);
5345 ASSURE_DESTINATION (charbuf_end
- charbuf
);
5346 while (charbuf
< charbuf_end
&& dst
< dst_end
)
5347 *dst
++ = *charbuf
++;
5349 produced_chars
= dst
- (coding
->destination
+ coding
->produced
);
5351 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
5352 coding
->produced_char
+= produced_chars
;
5353 coding
->produced
= dst
- coding
->destination
;
5357 /* See the above "GENERAL NOTES on `detect_coding_XXX ()' functions".
5358 Return true if a text is encoded in a charset-based coding system. */
5361 detect_coding_charset (struct coding_system
*coding
,
5362 struct coding_detection_info
*detect_info
)
5364 const unsigned char *src
= coding
->source
, *src_base
;
5365 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
5366 bool multibytep
= coding
->src_multibyte
;
5367 ptrdiff_t consumed_chars
= 0;
5368 Lisp_Object attrs
, valids
, name
;
5370 ptrdiff_t head_ascii
= coding
->head_ascii
;
5371 bool check_latin_extra
= 0;
5373 detect_info
->checked
|= CATEGORY_MASK_CHARSET
;
5375 coding
= &coding_categories
[coding_category_charset
];
5376 attrs
= CODING_ID_ATTRS (coding
->id
);
5377 valids
= AREF (attrs
, coding_attr_charset_valids
);
5378 name
= CODING_ID_NAME (coding
->id
);
5379 if (strncmp (SSDATA (SYMBOL_NAME (name
)),
5380 "iso-8859-", sizeof ("iso-8859-") - 1) == 0
5381 || strncmp (SSDATA (SYMBOL_NAME (name
)),
5382 "iso-latin-", sizeof ("iso-latin-") - 1) == 0)
5383 check_latin_extra
= 1;
5385 if (! NILP (CODING_ATTR_ASCII_COMPAT (attrs
)))
5392 struct charset
*charset
;
5399 val
= AREF (valids
, c
);
5405 && check_latin_extra
5406 && (!VECTORP (Vlatin_extra_code_table
)
5407 || NILP (AREF (Vlatin_extra_code_table
, c
))))
5409 found
= CATEGORY_MASK_CHARSET
;
5413 charset
= CHARSET_FROM_ID (XFASTINT (val
));
5414 dim
= CHARSET_DIMENSION (charset
);
5415 for (idx
= 1; idx
< dim
; idx
++)
5420 if (c
< charset
->code_space
[(dim
- 1 - idx
) * 4]
5421 || c
> charset
->code_space
[(dim
- 1 - idx
) * 4 + 1])
5430 for (; CONSP (val
); val
= XCDR (val
))
5432 charset
= CHARSET_FROM_ID (XFASTINT (XCAR (val
)));
5433 dim
= CHARSET_DIMENSION (charset
);
5439 if (c
< charset
->code_space
[(dim
- 1 - idx
) * 4]
5440 || c
> charset
->code_space
[(dim
- 1 - idx
) * 4 + 1])
5455 detect_info
->rejected
|= CATEGORY_MASK_CHARSET
;
5459 detect_info
->found
|= found
;
5464 decode_coding_charset (struct coding_system
*coding
)
5466 const unsigned char *src
= coding
->source
+ coding
->consumed
;
5467 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
5468 const unsigned char *src_base
;
5469 int *charbuf
= coding
->charbuf
+ coding
->charbuf_used
;
5470 /* We may produce one charset annotation in one loop and one more at
5473 = coding
->charbuf
+ coding
->charbuf_size
- (MAX_ANNOTATION_LENGTH
* 2);
5474 ptrdiff_t consumed_chars
= 0, consumed_chars_base
;
5475 bool multibytep
= coding
->src_multibyte
;
5476 Lisp_Object attrs
= CODING_ID_ATTRS (coding
->id
);
5478 ptrdiff_t char_offset
= coding
->produced_char
;
5479 ptrdiff_t last_offset
= char_offset
;
5480 int last_id
= charset_ascii
;
5482 = !inhibit_eol_conversion
&& EQ (CODING_ID_EOL_TYPE (coding
->id
), Qdos
);
5483 int byte_after_cr
= -1;
5485 valids
= AREF (attrs
, coding_attr_charset_valids
);
5491 struct charset
*charset
;
5497 consumed_chars_base
= consumed_chars
;
5499 if (charbuf
>= charbuf_end
)
5501 if (byte_after_cr
>= 0)
5506 if (byte_after_cr
>= 0)
5514 if (eol_dos
&& c
== '\r')
5515 ONE_MORE_BYTE (byte_after_cr
);
5521 val
= AREF (valids
, c
);
5522 if (! INTEGERP (val
) && ! CONSP (val
))
5526 charset
= CHARSET_FROM_ID (XFASTINT (val
));
5527 dim
= CHARSET_DIMENSION (charset
);
5531 code
= (code
<< 8) | c
;
5534 CODING_DECODE_CHAR (coding
, src
, src_base
, src_end
,
5539 /* VAL is a list of charset IDs. It is assured that the
5540 list is sorted by charset dimensions (smaller one
5544 charset
= CHARSET_FROM_ID (XFASTINT (XCAR (val
)));
5545 dim
= CHARSET_DIMENSION (charset
);
5549 code
= (code
<< 8) | c
;
5552 CODING_DECODE_CHAR (coding
, src
, src_base
,
5553 src_end
, charset
, code
, c
);
5561 if (charset
->id
!= charset_ascii
5562 && last_id
!= charset
->id
)
5564 if (last_id
!= charset_ascii
)
5565 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
5566 last_id
= charset
->id
;
5567 last_offset
= char_offset
;
5576 consumed_chars
= consumed_chars_base
;
5578 *charbuf
++ = c
< 0 ? -c
: ASCII_BYTE_P (c
) ? c
: BYTE8_TO_CHAR (c
);
5584 if (last_id
!= charset_ascii
)
5585 ADD_CHARSET_DATA (charbuf
, char_offset
- last_offset
, last_id
);
5586 coding
->consumed_char
+= consumed_chars_base
;
5587 coding
->consumed
= src_base
- coding
->source
;
5588 coding
->charbuf_used
= charbuf
- coding
->charbuf
;
5592 encode_coding_charset (struct coding_system
*coding
)
5594 bool multibytep
= coding
->dst_multibyte
;
5595 int *charbuf
= coding
->charbuf
;
5596 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
5597 unsigned char *dst
= coding
->destination
+ coding
->produced
;
5598 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
5599 int safe_room
= MAX_MULTIBYTE_LENGTH
;
5600 ptrdiff_t produced_chars
= 0;
5601 Lisp_Object attrs
, charset_list
;
5602 bool ascii_compatible
;
5605 CODING_GET_INFO (coding
, attrs
, charset_list
);
5606 ascii_compatible
= ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
));
5608 while (charbuf
< charbuf_end
)
5610 struct charset
*charset
;
5613 ASSURE_DESTINATION (safe_room
);
5615 if (ascii_compatible
&& ASCII_CHAR_P (c
))
5616 EMIT_ONE_ASCII_BYTE (c
);
5617 else if (CHAR_BYTE8_P (c
))
5619 c
= CHAR_TO_BYTE8 (c
);
5624 CODING_CHAR_CHARSET (coding
, dst
, dst_end
, c
, charset_list
,
5629 if (CHARSET_DIMENSION (charset
) == 1)
5630 EMIT_ONE_BYTE (code
);
5631 else if (CHARSET_DIMENSION (charset
) == 2)
5632 EMIT_TWO_BYTES (code
>> 8, code
& 0xFF);
5633 else if (CHARSET_DIMENSION (charset
) == 3)
5634 EMIT_THREE_BYTES (code
>> 16, (code
>> 8) & 0xFF, code
& 0xFF);
5636 EMIT_FOUR_BYTES (code
>> 24, (code
>> 16) & 0xFF,
5637 (code
>> 8) & 0xFF, code
& 0xFF);
5641 if (coding
->mode
& CODING_MODE_SAFE_ENCODING
)
5642 c
= CODING_INHIBIT_CHARACTER_SUBSTITUTION
;
5644 c
= coding
->default_char
;
5650 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
5651 coding
->produced_char
+= produced_chars
;
5652 coding
->produced
= dst
- coding
->destination
;
5657 /*** 7. C library functions ***/
5659 /* Setup coding context CODING from information about CODING_SYSTEM.
5660 If CODING_SYSTEM is nil, `no-conversion' is assumed. If
5661 CODING_SYSTEM is invalid, signal an error. */
5664 setup_coding_system (Lisp_Object coding_system
, struct coding_system
*coding
)
5667 Lisp_Object eol_type
;
5668 Lisp_Object coding_type
;
5671 if (NILP (coding_system
))
5672 coding_system
= Qundecided
;
5674 CHECK_CODING_SYSTEM_GET_ID (coding_system
, coding
->id
);
5676 attrs
= CODING_ID_ATTRS (coding
->id
);
5677 eol_type
= inhibit_eol_conversion
? Qunix
: CODING_ID_EOL_TYPE (coding
->id
);
5680 if (VECTORP (eol_type
))
5681 coding
->common_flags
= (CODING_REQUIRE_DECODING_MASK
5682 | CODING_REQUIRE_DETECTION_MASK
);
5683 else if (! EQ (eol_type
, Qunix
))
5684 coding
->common_flags
= (CODING_REQUIRE_DECODING_MASK
5685 | CODING_REQUIRE_ENCODING_MASK
);
5687 coding
->common_flags
= 0;
5688 if (! NILP (CODING_ATTR_POST_READ (attrs
)))
5689 coding
->common_flags
|= CODING_REQUIRE_DECODING_MASK
;
5690 if (! NILP (CODING_ATTR_PRE_WRITE (attrs
)))
5691 coding
->common_flags
|= CODING_REQUIRE_ENCODING_MASK
;
5692 if (! NILP (CODING_ATTR_FOR_UNIBYTE (attrs
)))
5693 coding
->common_flags
|= CODING_FOR_UNIBYTE_MASK
;
5695 val
= CODING_ATTR_SAFE_CHARSETS (attrs
);
5696 coding
->max_charset_id
= SCHARS (val
) - 1;
5697 coding
->safe_charsets
= SDATA (val
);
5698 coding
->default_char
= XINT (CODING_ATTR_DEFAULT_CHAR (attrs
));
5699 coding
->carryover_bytes
= 0;
5701 coding_type
= CODING_ATTR_TYPE (attrs
);
5702 if (EQ (coding_type
, Qundecided
))
5704 coding
->detector
= NULL
;
5705 coding
->decoder
= decode_coding_raw_text
;
5706 coding
->encoder
= encode_coding_raw_text
;
5707 coding
->common_flags
|= CODING_REQUIRE_DETECTION_MASK
;
5708 coding
->spec
.undecided
.inhibit_nbd
5709 = (NILP (AREF (attrs
, coding_attr_undecided_inhibit_null_byte_detection
))
5711 : EQ (AREF (attrs
, coding_attr_undecided_inhibit_null_byte_detection
), Qt
)
5714 coding
->spec
.undecided
.inhibit_ied
5715 = (NILP (AREF (attrs
, coding_attr_undecided_inhibit_iso_escape_detection
))
5717 : EQ (AREF (attrs
, coding_attr_undecided_inhibit_iso_escape_detection
), Qt
)
5720 coding
->spec
.undecided
.prefer_utf_8
5721 = ! NILP (AREF (attrs
, coding_attr_undecided_prefer_utf_8
));
5723 else if (EQ (coding_type
, Qiso_2022
))
5726 int flags
= XINT (AREF (attrs
, coding_attr_iso_flags
));
5728 /* Invoke graphic register 0 to plane 0. */
5729 CODING_ISO_INVOCATION (coding
, 0) = 0;
5730 /* Invoke graphic register 1 to plane 1 if we can use 8-bit. */
5731 CODING_ISO_INVOCATION (coding
, 1)
5732 = (flags
& CODING_ISO_FLAG_SEVEN_BITS
? -1 : 1);
5733 /* Setup the initial status of designation. */
5734 for (i
= 0; i
< 4; i
++)
5735 CODING_ISO_DESIGNATION (coding
, i
) = CODING_ISO_INITIAL (coding
, i
);
5736 /* Not single shifting initially. */
5737 CODING_ISO_SINGLE_SHIFTING (coding
) = 0;
5738 /* Beginning of buffer should also be regarded as bol. */
5739 CODING_ISO_BOL (coding
) = 1;
5740 coding
->detector
= detect_coding_iso_2022
;
5741 coding
->decoder
= decode_coding_iso_2022
;
5742 coding
->encoder
= encode_coding_iso_2022
;
5743 if (flags
& CODING_ISO_FLAG_SAFE
)
5744 coding
->mode
|= CODING_MODE_SAFE_ENCODING
;
5745 coding
->common_flags
5746 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
5747 | CODING_REQUIRE_FLUSHING_MASK
);
5748 if (flags
& CODING_ISO_FLAG_COMPOSITION
)
5749 coding
->common_flags
|= CODING_ANNOTATE_COMPOSITION_MASK
;
5750 if (flags
& CODING_ISO_FLAG_DESIGNATION
)
5751 coding
->common_flags
|= CODING_ANNOTATE_CHARSET_MASK
;
5752 if (flags
& CODING_ISO_FLAG_FULL_SUPPORT
)
5754 setup_iso_safe_charsets (attrs
);
5755 val
= CODING_ATTR_SAFE_CHARSETS (attrs
);
5756 coding
->max_charset_id
= SCHARS (val
) - 1;
5757 coding
->safe_charsets
= SDATA (val
);
5759 CODING_ISO_FLAGS (coding
) = flags
;
5760 CODING_ISO_CMP_STATUS (coding
)->state
= COMPOSING_NO
;
5761 CODING_ISO_CMP_STATUS (coding
)->method
= COMPOSITION_NO
;
5762 CODING_ISO_EXTSEGMENT_LEN (coding
) = 0;
5763 CODING_ISO_EMBEDDED_UTF_8 (coding
) = 0;
5765 else if (EQ (coding_type
, Qcharset
))
5767 coding
->detector
= detect_coding_charset
;
5768 coding
->decoder
= decode_coding_charset
;
5769 coding
->encoder
= encode_coding_charset
;
5770 coding
->common_flags
5771 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
);
5773 else if (EQ (coding_type
, Qutf_8
))
5775 val
= AREF (attrs
, coding_attr_utf_bom
);
5776 CODING_UTF_8_BOM (coding
) = (CONSP (val
) ? utf_detect_bom
5777 : EQ (val
, Qt
) ? utf_with_bom
5779 coding
->detector
= detect_coding_utf_8
;
5780 coding
->decoder
= decode_coding_utf_8
;
5781 coding
->encoder
= encode_coding_utf_8
;
5782 coding
->common_flags
5783 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
);
5784 if (CODING_UTF_8_BOM (coding
) == utf_detect_bom
)
5785 coding
->common_flags
|= CODING_REQUIRE_DETECTION_MASK
;
5787 else if (EQ (coding_type
, Qutf_16
))
5789 val
= AREF (attrs
, coding_attr_utf_bom
);
5790 CODING_UTF_16_BOM (coding
) = (CONSP (val
) ? utf_detect_bom
5791 : EQ (val
, Qt
) ? utf_with_bom
5793 val
= AREF (attrs
, coding_attr_utf_16_endian
);
5794 CODING_UTF_16_ENDIAN (coding
) = (EQ (val
, Qbig
) ? utf_16_big_endian
5795 : utf_16_little_endian
);
5796 CODING_UTF_16_SURROGATE (coding
) = 0;
5797 coding
->detector
= detect_coding_utf_16
;
5798 coding
->decoder
= decode_coding_utf_16
;
5799 coding
->encoder
= encode_coding_utf_16
;
5800 coding
->common_flags
5801 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
);
5802 if (CODING_UTF_16_BOM (coding
) == utf_detect_bom
)
5803 coding
->common_flags
|= CODING_REQUIRE_DETECTION_MASK
;
5805 else if (EQ (coding_type
, Qccl
))
5807 coding
->detector
= detect_coding_ccl
;
5808 coding
->decoder
= decode_coding_ccl
;
5809 coding
->encoder
= encode_coding_ccl
;
5810 coding
->common_flags
5811 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
5812 | CODING_REQUIRE_FLUSHING_MASK
);
5814 else if (EQ (coding_type
, Qemacs_mule
))
5816 coding
->detector
= detect_coding_emacs_mule
;
5817 coding
->decoder
= decode_coding_emacs_mule
;
5818 coding
->encoder
= encode_coding_emacs_mule
;
5819 coding
->common_flags
5820 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
);
5821 if (! NILP (AREF (attrs
, coding_attr_emacs_mule_full
))
5822 && ! EQ (CODING_ATTR_CHARSET_LIST (attrs
), Vemacs_mule_charset_list
))
5824 Lisp_Object tail
, safe_charsets
;
5825 int max_charset_id
= 0;
5827 for (tail
= Vemacs_mule_charset_list
; CONSP (tail
);
5829 if (max_charset_id
< XFASTINT (XCAR (tail
)))
5830 max_charset_id
= XFASTINT (XCAR (tail
));
5831 safe_charsets
= make_uninit_string (max_charset_id
+ 1);
5832 memset (SDATA (safe_charsets
), 255, max_charset_id
+ 1);
5833 for (tail
= Vemacs_mule_charset_list
; CONSP (tail
);
5835 SSET (safe_charsets
, XFASTINT (XCAR (tail
)), 0);
5836 coding
->max_charset_id
= max_charset_id
;
5837 coding
->safe_charsets
= SDATA (safe_charsets
);
5839 coding
->spec
.emacs_mule
.cmp_status
.state
= COMPOSING_NO
;
5840 coding
->spec
.emacs_mule
.cmp_status
.method
= COMPOSITION_NO
;
5842 else if (EQ (coding_type
, Qshift_jis
))
5844 coding
->detector
= detect_coding_sjis
;
5845 coding
->decoder
= decode_coding_sjis
;
5846 coding
->encoder
= encode_coding_sjis
;
5847 coding
->common_flags
5848 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
);
5850 else if (EQ (coding_type
, Qbig5
))
5852 coding
->detector
= detect_coding_big5
;
5853 coding
->decoder
= decode_coding_big5
;
5854 coding
->encoder
= encode_coding_big5
;
5855 coding
->common_flags
5856 |= (CODING_REQUIRE_DECODING_MASK
| CODING_REQUIRE_ENCODING_MASK
);
5858 else /* EQ (coding_type, Qraw_text) */
5860 coding
->detector
= NULL
;
5861 coding
->decoder
= decode_coding_raw_text
;
5862 coding
->encoder
= encode_coding_raw_text
;
5863 if (! EQ (eol_type
, Qunix
))
5865 coding
->common_flags
|= CODING_REQUIRE_DECODING_MASK
;
5866 if (! VECTORP (eol_type
))
5867 coding
->common_flags
|= CODING_REQUIRE_ENCODING_MASK
;
5875 /* Return a list of charsets supported by CODING. */
5878 coding_charset_list (struct coding_system
*coding
)
5880 Lisp_Object attrs
, charset_list
;
5882 CODING_GET_INFO (coding
, attrs
, charset_list
);
5883 if (EQ (CODING_ATTR_TYPE (attrs
), Qiso_2022
))
5885 int flags
= XINT (AREF (attrs
, coding_attr_iso_flags
));
5887 if (flags
& CODING_ISO_FLAG_FULL_SUPPORT
)
5888 charset_list
= Viso_2022_charset_list
;
5890 else if (EQ (CODING_ATTR_TYPE (attrs
), Qemacs_mule
))
5892 charset_list
= Vemacs_mule_charset_list
;
5894 return charset_list
;
5898 /* Return a list of charsets supported by CODING-SYSTEM. */
5901 coding_system_charset_list (Lisp_Object coding_system
)
5904 Lisp_Object attrs
, charset_list
;
5906 CHECK_CODING_SYSTEM_GET_ID (coding_system
, id
);
5907 attrs
= CODING_ID_ATTRS (id
);
5909 if (EQ (CODING_ATTR_TYPE (attrs
), Qiso_2022
))
5911 int flags
= XINT (AREF (attrs
, coding_attr_iso_flags
));
5913 if (flags
& CODING_ISO_FLAG_FULL_SUPPORT
)
5914 charset_list
= Viso_2022_charset_list
;
5916 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
5918 else if (EQ (CODING_ATTR_TYPE (attrs
), Qemacs_mule
))
5920 charset_list
= Vemacs_mule_charset_list
;
5924 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
5926 return charset_list
;
5930 /* Return raw-text or one of its subsidiaries that has the same
5931 eol_type as CODING-SYSTEM. */
5934 raw_text_coding_system (Lisp_Object coding_system
)
5936 Lisp_Object spec
, attrs
;
5937 Lisp_Object eol_type
, raw_text_eol_type
;
5939 if (NILP (coding_system
))
5941 spec
= CODING_SYSTEM_SPEC (coding_system
);
5942 attrs
= AREF (spec
, 0);
5944 if (EQ (CODING_ATTR_TYPE (attrs
), Qraw_text
))
5945 return coding_system
;
5947 eol_type
= AREF (spec
, 2);
5948 if (VECTORP (eol_type
))
5950 spec
= CODING_SYSTEM_SPEC (Qraw_text
);
5951 raw_text_eol_type
= AREF (spec
, 2);
5952 return (EQ (eol_type
, Qunix
) ? AREF (raw_text_eol_type
, 0)
5953 : EQ (eol_type
, Qdos
) ? AREF (raw_text_eol_type
, 1)
5954 : AREF (raw_text_eol_type
, 2));
5958 /* If CODING_SYSTEM doesn't specify end-of-line format, return one of
5959 the subsidiary that has the same eol-spec as PARENT (if it is not
5960 nil and specifies end-of-line format) or the system's setting
5961 (system_eol_type). */
5964 coding_inherit_eol_type (Lisp_Object coding_system
, Lisp_Object parent
)
5966 Lisp_Object spec
, eol_type
;
5968 if (NILP (coding_system
))
5969 coding_system
= Qraw_text
;
5970 spec
= CODING_SYSTEM_SPEC (coding_system
);
5971 eol_type
= AREF (spec
, 2);
5972 if (VECTORP (eol_type
))
5974 Lisp_Object parent_eol_type
;
5976 if (! NILP (parent
))
5978 Lisp_Object parent_spec
;
5980 parent_spec
= CODING_SYSTEM_SPEC (parent
);
5981 parent_eol_type
= AREF (parent_spec
, 2);
5982 if (VECTORP (parent_eol_type
))
5983 parent_eol_type
= system_eol_type
;
5986 parent_eol_type
= system_eol_type
;
5987 if (EQ (parent_eol_type
, Qunix
))
5988 coding_system
= AREF (eol_type
, 0);
5989 else if (EQ (parent_eol_type
, Qdos
))
5990 coding_system
= AREF (eol_type
, 1);
5991 else if (EQ (parent_eol_type
, Qmac
))
5992 coding_system
= AREF (eol_type
, 2);
5994 return coding_system
;
5998 /* Check if text-conversion and eol-conversion of CODING_SYSTEM are
5999 decided for writing to a process. If not, complement them, and
6000 return a new coding system. */
6003 complement_process_encoding_system (Lisp_Object coding_system
)
6005 Lisp_Object coding_base
= Qnil
, eol_base
= Qnil
;
6006 Lisp_Object spec
, attrs
;
6009 for (i
= 0; i
< 3; i
++)
6012 coding_system
= CDR_SAFE (Vdefault_process_coding_system
);
6014 coding_system
= preferred_coding_system ();
6015 spec
= CODING_SYSTEM_SPEC (coding_system
);
6018 attrs
= AREF (spec
, 0);
6019 if (NILP (coding_base
) && ! EQ (CODING_ATTR_TYPE (attrs
), Qundecided
))
6020 coding_base
= CODING_ATTR_BASE_NAME (attrs
);
6021 if (NILP (eol_base
) && ! VECTORP (AREF (spec
, 2)))
6022 eol_base
= coding_system
;
6023 if (! NILP (coding_base
) && ! NILP (eol_base
))
6028 /* The original CODING_SYSTEM didn't specify text-conversion or
6029 eol-conversion. Be sure that we return a fully complemented
6031 coding_system
= coding_inherit_eol_type (coding_base
, eol_base
);
6032 return coding_system
;
6036 /* Emacs has a mechanism to automatically detect a coding system if it
6037 is one of Emacs' internal format, ISO2022, SJIS, and BIG5. But,
6038 it's impossible to distinguish some coding systems accurately
6039 because they use the same range of codes. So, at first, coding
6040 systems are categorized into 7, those are:
6042 o coding-category-emacs-mule
6044 The category for a coding system which has the same code range
6045 as Emacs' internal format. Assigned the coding-system (Lisp
6046 symbol) `emacs-mule' by default.
6048 o coding-category-sjis
6050 The category for a coding system which has the same code range
6051 as SJIS. Assigned the coding-system (Lisp
6052 symbol) `japanese-shift-jis' by default.
6054 o coding-category-iso-7
6056 The category for a coding system which has the same code range
6057 as ISO2022 of 7-bit environment. This doesn't use any locking
6058 shift and single shift functions. This can encode/decode all
6059 charsets. Assigned the coding-system (Lisp symbol)
6060 `iso-2022-7bit' by default.
6062 o coding-category-iso-7-tight
6064 Same as coding-category-iso-7 except that this can
6065 encode/decode only the specified charsets.
6067 o coding-category-iso-8-1
6069 The category for a coding system which has the same code range
6070 as ISO2022 of 8-bit environment and graphic plane 1 used only
6071 for DIMENSION1 charset. This doesn't use any locking shift
6072 and single shift functions. Assigned the coding-system (Lisp
6073 symbol) `iso-latin-1' by default.
6075 o coding-category-iso-8-2
6077 The category for a coding system which has the same code range
6078 as ISO2022 of 8-bit environment and graphic plane 1 used only
6079 for DIMENSION2 charset. This doesn't use any locking shift
6080 and single shift functions. Assigned the coding-system (Lisp
6081 symbol) `japanese-iso-8bit' by default.
6083 o coding-category-iso-7-else
6085 The category for a coding system which has the same code range
6086 as ISO2022 of 7-bit environment but uses locking shift or
6087 single shift functions. Assigned the coding-system (Lisp
6088 symbol) `iso-2022-7bit-lock' by default.
6090 o coding-category-iso-8-else
6092 The category for a coding system which has the same code range
6093 as ISO2022 of 8-bit environment but uses locking shift or
6094 single shift functions. Assigned the coding-system (Lisp
6095 symbol) `iso-2022-8bit-ss2' by default.
6097 o coding-category-big5
6099 The category for a coding system which has the same code range
6100 as BIG5. Assigned the coding-system (Lisp symbol)
6101 `cn-big5' by default.
6103 o coding-category-utf-8
6105 The category for a coding system which has the same code range
6106 as UTF-8 (cf. RFC3629). Assigned the coding-system (Lisp
6107 symbol) `utf-8' by default.
6109 o coding-category-utf-16-be
6111 The category for a coding system in which a text has an
6112 Unicode signature (cf. Unicode Standard) in the order of BIG
6113 endian at the head. Assigned the coding-system (Lisp symbol)
6114 `utf-16-be' by default.
6116 o coding-category-utf-16-le
6118 The category for a coding system in which a text has an
6119 Unicode signature (cf. Unicode Standard) in the order of
6120 LITTLE endian at the head. Assigned the coding-system (Lisp
6121 symbol) `utf-16-le' by default.
6123 o coding-category-ccl
6125 The category for a coding system of which encoder/decoder is
6126 written in CCL programs. The default value is nil, i.e., no
6127 coding system is assigned.
6129 o coding-category-binary
6131 The category for a coding system not categorized in any of the
6132 above. Assigned the coding-system (Lisp symbol)
6133 `no-conversion' by default.
6135 Each of them is a Lisp symbol and the value is an actual
6136 `coding-system's (this is also a Lisp symbol) assigned by a user.
6137 What Emacs does actually is to detect a category of coding system.
6138 Then, it uses a `coding-system' assigned to it. If Emacs can't
6139 decide only one possible category, it selects a category of the
6140 highest priority. Priorities of categories are also specified by a
6141 user in a Lisp variable `coding-category-list'.
6145 static Lisp_Object
adjust_coding_eol_type (struct coding_system
*coding
,
6149 /* Return the number of ASCII characters at the head of the source.
6150 By side effects, set coding->head_ascii and update
6151 coding->eol_seen. The value of coding->eol_seen is "logical or" of
6152 EOL_SEEN_LF, EOL_SEEN_CR, and EOL_SEEN_CRLF, but the value is
6153 reliable only when all the source bytes are ASCII. */
6156 check_ascii (struct coding_system
*coding
)
6158 const unsigned char *src
, *end
;
6159 Lisp_Object eol_type
= CODING_ID_EOL_TYPE (coding
->id
);
6160 int eol_seen
= coding
->eol_seen
;
6162 coding_set_source (coding
);
6163 src
= coding
->source
;
6164 end
= src
+ coding
->src_bytes
;
6166 if (inhibit_eol_conversion
6167 || SYMBOLP (eol_type
))
6169 /* We don't have to check EOL format. */
6170 while (src
< end
&& !( *src
& 0x80))
6173 eol_seen
|= EOL_SEEN_LF
;
6178 end
--; /* We look ahead one byte for "CR LF". */
6190 eol_seen
|= EOL_SEEN_CRLF
;
6194 eol_seen
|= EOL_SEEN_CR
;
6197 eol_seen
|= EOL_SEEN_LF
;
6203 /* All bytes but the last one C are ASCII. */
6207 eol_seen
|= EOL_SEEN_CR
;
6209 eol_seen
|= EOL_SEEN_LF
;
6214 coding
->head_ascii
= src
- coding
->source
;
6215 coding
->eol_seen
= eol_seen
;
6216 return (coding
->head_ascii
);
6220 /* Return the number of characters at the source if all the bytes are
6221 valid UTF-8 (of Unicode range). Otherwise, return -1. By side
6222 effects, update coding->eol_seen. The value of coding->eol_seen is
6223 "logical or" of EOL_SEEN_LF, EOL_SEEN_CR, and EOL_SEEN_CRLF, but
6224 the value is reliable only when all the source bytes are valid
6228 check_utf_8 (struct coding_system
*coding
)
6230 const unsigned char *src
, *end
;
6232 int nchars
= coding
->head_ascii
;
6234 if (coding
->head_ascii
< 0)
6235 check_ascii (coding
);
6237 coding_set_source (coding
);
6238 src
= coding
->source
+ coding
->head_ascii
;
6239 /* We look ahead one byte for CR LF. */
6240 end
= coding
->source
+ coding
->src_bytes
- 1;
6241 eol_seen
= coding
->eol_seen
;
6246 if (UTF_8_1_OCTET_P (*src
))
6255 eol_seen
|= EOL_SEEN_CRLF
;
6260 eol_seen
|= EOL_SEEN_CR
;
6263 eol_seen
|= EOL_SEEN_LF
;
6266 else if (UTF_8_2_OCTET_LEADING_P (c
))
6268 if (c
< 0xC2 /* overlong sequence */
6270 || ! UTF_8_EXTRA_OCTET_P (src
[1]))
6274 else if (UTF_8_3_OCTET_LEADING_P (c
))
6277 || ! (UTF_8_EXTRA_OCTET_P (src
[1])
6278 && UTF_8_EXTRA_OCTET_P (src
[2])))
6280 c
= (((c
& 0xF) << 12)
6281 | ((src
[1] & 0x3F) << 6) | (src
[2] & 0x3F));
6282 if (c
< 0x800 /* overlong sequence */
6283 || (c
>= 0xd800 && c
< 0xe000)) /* surrogates (invalid) */
6287 else if (UTF_8_4_OCTET_LEADING_P (c
))
6290 || ! (UTF_8_EXTRA_OCTET_P (src
[1])
6291 && UTF_8_EXTRA_OCTET_P (src
[2])
6292 && UTF_8_EXTRA_OCTET_P (src
[3])))
6294 c
= (((c
& 0x7) << 18) | ((src
[1] & 0x3F) << 12)
6295 | ((src
[2] & 0x3F) << 6) | (src
[3] & 0x3F));
6296 if (c
< 0x10000 /* overlong sequence */
6297 || c
>= 0x110000) /* non-Unicode character */
6308 if (! UTF_8_1_OCTET_P (*src
))
6312 eol_seen
|= EOL_SEEN_CR
;
6313 else if (*src
== '\n')
6314 eol_seen
|= EOL_SEEN_LF
;
6316 coding
->eol_seen
= eol_seen
;
6321 /* Detect how end-of-line of a text of length SRC_BYTES pointed by
6322 SOURCE is encoded. If CATEGORY is one of
6323 coding_category_utf_16_XXXX, assume that CR and LF are encoded by
6324 two-byte, else they are encoded by one-byte.
6326 Return one of EOL_SEEN_XXX. */
6328 #define MAX_EOL_CHECK_COUNT 3
6331 detect_eol (const unsigned char *source
, ptrdiff_t src_bytes
,
6332 enum coding_category category
)
6334 const unsigned char *src
= source
, *src_end
= src
+ src_bytes
;
6337 int eol_seen
= EOL_SEEN_NONE
;
6339 if ((1 << category
) & CATEGORY_MASK_UTF_16
)
6341 bool msb
= category
== (coding_category_utf_16_le
6342 | coding_category_utf_16_le_nosig
);
6345 while (src
+ 1 < src_end
)
6348 if (src
[msb
] == 0 && (c
== '\n' || c
== '\r'))
6353 this_eol
= EOL_SEEN_LF
;
6354 else if (src
+ 3 >= src_end
6355 || src
[msb
+ 2] != 0
6356 || src
[lsb
+ 2] != '\n')
6357 this_eol
= EOL_SEEN_CR
;
6360 this_eol
= EOL_SEEN_CRLF
;
6364 if (eol_seen
== EOL_SEEN_NONE
)
6365 /* This is the first end-of-line. */
6366 eol_seen
= this_eol
;
6367 else if (eol_seen
!= this_eol
)
6369 /* The found type is different from what found before.
6370 Allow for stray ^M characters in DOS EOL files. */
6371 if ((eol_seen
== EOL_SEEN_CR
&& this_eol
== EOL_SEEN_CRLF
)
6372 || (eol_seen
== EOL_SEEN_CRLF
6373 && this_eol
== EOL_SEEN_CR
))
6374 eol_seen
= EOL_SEEN_CRLF
;
6377 eol_seen
= EOL_SEEN_LF
;
6381 if (++total
== MAX_EOL_CHECK_COUNT
)
6388 while (src
< src_end
)
6391 if (c
== '\n' || c
== '\r')
6396 this_eol
= EOL_SEEN_LF
;
6397 else if (src
>= src_end
|| *src
!= '\n')
6398 this_eol
= EOL_SEEN_CR
;
6400 this_eol
= EOL_SEEN_CRLF
, src
++;
6402 if (eol_seen
== EOL_SEEN_NONE
)
6403 /* This is the first end-of-line. */
6404 eol_seen
= this_eol
;
6405 else if (eol_seen
!= this_eol
)
6407 /* The found type is different from what found before.
6408 Allow for stray ^M characters in DOS EOL files. */
6409 if ((eol_seen
== EOL_SEEN_CR
&& this_eol
== EOL_SEEN_CRLF
)
6410 || (eol_seen
== EOL_SEEN_CRLF
&& this_eol
== EOL_SEEN_CR
))
6411 eol_seen
= EOL_SEEN_CRLF
;
6414 eol_seen
= EOL_SEEN_LF
;
6418 if (++total
== MAX_EOL_CHECK_COUNT
)
6427 adjust_coding_eol_type (struct coding_system
*coding
, int eol_seen
)
6429 Lisp_Object eol_type
;
6431 eol_type
= CODING_ID_EOL_TYPE (coding
->id
);
6432 if (! VECTORP (eol_type
))
6433 /* Already adjusted. */
6435 if (eol_seen
& EOL_SEEN_LF
)
6437 coding
->id
= CODING_SYSTEM_ID (AREF (eol_type
, 0));
6440 else if (eol_seen
& EOL_SEEN_CRLF
)
6442 coding
->id
= CODING_SYSTEM_ID (AREF (eol_type
, 1));
6445 else if (eol_seen
& EOL_SEEN_CR
)
6447 coding
->id
= CODING_SYSTEM_ID (AREF (eol_type
, 2));
6453 /* Detect how a text specified in CODING is encoded. If a coding
6454 system is detected, update fields of CODING by the detected coding
6458 detect_coding (struct coding_system
*coding
)
6460 const unsigned char *src
, *src_end
;
6461 unsigned int saved_mode
= coding
->mode
;
6462 Lisp_Object found
= Qnil
;
6463 Lisp_Object eol_type
= CODING_ID_EOL_TYPE (coding
->id
);
6465 coding
->consumed
= coding
->consumed_char
= 0;
6466 coding
->produced
= coding
->produced_char
= 0;
6467 coding_set_source (coding
);
6469 src_end
= coding
->source
+ coding
->src_bytes
;
6471 coding
->eol_seen
= EOL_SEEN_NONE
;
6472 /* If we have not yet decided the text encoding type, detect it
6474 if (EQ (CODING_ATTR_TYPE (CODING_ID_ATTRS (coding
->id
)), Qundecided
))
6477 struct coding_detection_info detect_info
;
6478 bool null_byte_found
= 0, eight_bit_found
= 0;
6479 int inhibit_nbd
/* null byte detection */
6480 = (coding
->spec
.undecided
.inhibit_nbd
> 0
6481 | (coding
->spec
.undecided
.inhibit_nbd
== 0
6482 & inhibit_null_byte_detection
));
6483 int inhibit_ied
/* iso escape detection */
6484 = (coding
->spec
.undecided
.inhibit_ied
> 0
6485 | (coding
->spec
.undecided
.inhibit_ied
== 0
6486 & inhibit_iso_escape_detection
));
6488 = coding
->spec
.undecided
.prefer_utf_8
;
6490 coding
->head_ascii
= 0;
6491 detect_info
.checked
= detect_info
.found
= detect_info
.rejected
= 0;
6492 for (src
= coding
->source
; src
< src_end
; src
++)
6497 eight_bit_found
= 1;
6498 if (null_byte_found
)
6503 if ((c
== ISO_CODE_ESC
|| c
== ISO_CODE_SI
|| c
== ISO_CODE_SO
)
6505 && ! detect_info
.checked
)
6507 if (detect_coding_iso_2022 (coding
, &detect_info
))
6509 /* We have scanned the whole data. */
6510 if (! (detect_info
.rejected
& CATEGORY_MASK_ISO_7_ELSE
))
6512 /* We didn't find an 8-bit code. We may
6513 have found a null-byte, but it's very
6514 rare that a binary file conforms to
6517 coding
->head_ascii
= src
- coding
->source
;
6519 detect_info
.rejected
|= ~CATEGORY_MASK_ISO_ESCAPE
;
6523 else if (! c
&& !inhibit_nbd
)
6525 null_byte_found
= 1;
6526 if (eight_bit_found
)
6529 else if (! disable_ascii_optimization
6530 && ! inhibit_eol_conversion
)
6534 if (src
< src_end
&& src
[1] == '\n')
6536 coding
->eol_seen
|= EOL_SEEN_CRLF
;
6538 if (! eight_bit_found
)
6539 coding
->head_ascii
++;
6542 coding
->eol_seen
|= EOL_SEEN_CR
;
6546 coding
->eol_seen
|= EOL_SEEN_LF
;
6550 if (! eight_bit_found
)
6551 coding
->head_ascii
++;
6553 else if (! eight_bit_found
)
6554 coding
->head_ascii
++;
6557 if (null_byte_found
|| eight_bit_found
6558 || coding
->head_ascii
< coding
->src_bytes
6559 || detect_info
.found
)
6561 enum coding_category category
;
6562 struct coding_system
*this;
6564 if (coding
->head_ascii
== coding
->src_bytes
)
6565 /* As all bytes are 7-bit, we can ignore non-ISO-2022 codings. */
6566 for (i
= 0; i
< coding_category_raw_text
; i
++)
6568 category
= coding_priorities
[i
];
6569 this = coding_categories
+ category
;
6570 if (detect_info
.found
& (1 << category
))
6575 if (null_byte_found
)
6577 detect_info
.checked
|= ~CATEGORY_MASK_UTF_16
;
6578 detect_info
.rejected
|= ~CATEGORY_MASK_UTF_16
;
6580 else if (prefer_utf_8
6581 && detect_coding_utf_8 (coding
, &detect_info
))
6583 detect_info
.checked
|= ~CATEGORY_MASK_UTF_8
;
6584 detect_info
.rejected
|= ~CATEGORY_MASK_UTF_8
;
6586 for (i
= 0; i
< coding_category_raw_text
; i
++)
6588 category
= coding_priorities
[i
];
6589 this = coding_categories
+ category
;
6590 /* Some of this->detector (e.g. detect_coding_sjis)
6591 require this information. */
6592 coding
->id
= this->id
;
6595 /* No coding system of this category is defined. */
6596 detect_info
.rejected
|= (1 << category
);
6598 else if (category
>= coding_category_raw_text
)
6600 else if (detect_info
.checked
& (1 << category
))
6602 if (detect_info
.found
& (1 << category
))
6605 else if ((*(this->detector
)) (coding
, &detect_info
)
6606 && detect_info
.found
& (1 << category
))
6611 if (i
< coding_category_raw_text
)
6613 if (category
== coding_category_utf_8_auto
)
6615 Lisp_Object coding_systems
;
6617 coding_systems
= AREF (CODING_ID_ATTRS (this->id
),
6618 coding_attr_utf_bom
);
6619 if (CONSP (coding_systems
))
6621 if (detect_info
.found
& CATEGORY_MASK_UTF_8_SIG
)
6622 found
= XCAR (coding_systems
);
6624 found
= XCDR (coding_systems
);
6627 found
= CODING_ID_NAME (this->id
);
6629 else if (category
== coding_category_utf_16_auto
)
6631 Lisp_Object coding_systems
;
6633 coding_systems
= AREF (CODING_ID_ATTRS (this->id
),
6634 coding_attr_utf_bom
);
6635 if (CONSP (coding_systems
))
6637 if (detect_info
.found
& CATEGORY_MASK_UTF_16_LE
)
6638 found
= XCAR (coding_systems
);
6639 else if (detect_info
.found
& CATEGORY_MASK_UTF_16_BE
)
6640 found
= XCDR (coding_systems
);
6643 found
= CODING_ID_NAME (this->id
);
6646 found
= CODING_ID_NAME (this->id
);
6648 else if (null_byte_found
)
6649 found
= Qno_conversion
;
6650 else if ((detect_info
.rejected
& CATEGORY_MASK_ANY
)
6651 == CATEGORY_MASK_ANY
)
6653 else if (detect_info
.rejected
)
6654 for (i
= 0; i
< coding_category_raw_text
; i
++)
6655 if (! (detect_info
.rejected
& (1 << coding_priorities
[i
])))
6657 this = coding_categories
+ coding_priorities
[i
];
6658 found
= CODING_ID_NAME (this->id
);
6663 else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding
->id
)))
6664 == coding_category_utf_8_auto
)
6666 Lisp_Object coding_systems
;
6667 struct coding_detection_info detect_info
;
6670 = AREF (CODING_ID_ATTRS (coding
->id
), coding_attr_utf_bom
);
6671 detect_info
.found
= detect_info
.rejected
= 0;
6672 if (check_ascii (coding
) == coding
->src_bytes
)
6674 if (CONSP (coding_systems
))
6675 found
= XCDR (coding_systems
);
6679 if (CONSP (coding_systems
)
6680 && detect_coding_utf_8 (coding
, &detect_info
))
6682 if (detect_info
.found
& CATEGORY_MASK_UTF_8_SIG
)
6683 found
= XCAR (coding_systems
);
6685 found
= XCDR (coding_systems
);
6689 else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding
->id
)))
6690 == coding_category_utf_16_auto
)
6692 Lisp_Object coding_systems
;
6693 struct coding_detection_info detect_info
;
6696 = AREF (CODING_ID_ATTRS (coding
->id
), coding_attr_utf_bom
);
6697 detect_info
.found
= detect_info
.rejected
= 0;
6698 coding
->head_ascii
= 0;
6699 if (CONSP (coding_systems
)
6700 && detect_coding_utf_16 (coding
, &detect_info
))
6702 if (detect_info
.found
& CATEGORY_MASK_UTF_16_LE
)
6703 found
= XCAR (coding_systems
);
6704 else if (detect_info
.found
& CATEGORY_MASK_UTF_16_BE
)
6705 found
= XCDR (coding_systems
);
6711 int specified_eol
= (VECTORP (eol_type
) ? EOL_SEEN_NONE
6712 : EQ (eol_type
, Qdos
) ? EOL_SEEN_CRLF
6713 : EQ (eol_type
, Qmac
) ? EOL_SEEN_CR
6716 setup_coding_system (found
, coding
);
6717 if (specified_eol
!= EOL_SEEN_NONE
)
6718 adjust_coding_eol_type (coding
, specified_eol
);
6721 coding
->mode
= saved_mode
;
6726 decode_eol (struct coding_system
*coding
)
6728 Lisp_Object eol_type
;
6729 unsigned char *p
, *pbeg
, *pend
;
6731 eol_type
= CODING_ID_EOL_TYPE (coding
->id
);
6732 if (EQ (eol_type
, Qunix
) || inhibit_eol_conversion
)
6735 if (NILP (coding
->dst_object
))
6736 pbeg
= coding
->destination
;
6738 pbeg
= BYTE_POS_ADDR (coding
->dst_pos_byte
);
6739 pend
= pbeg
+ coding
->produced
;
6741 if (VECTORP (eol_type
))
6743 int eol_seen
= EOL_SEEN_NONE
;
6745 for (p
= pbeg
; p
< pend
; p
++)
6748 eol_seen
|= EOL_SEEN_LF
;
6749 else if (*p
== '\r')
6751 if (p
+ 1 < pend
&& *(p
+ 1) == '\n')
6753 eol_seen
|= EOL_SEEN_CRLF
;
6757 eol_seen
|= EOL_SEEN_CR
;
6760 /* Handle DOS-style EOLs in a file with stray ^M characters. */
6761 if ((eol_seen
& EOL_SEEN_CRLF
) != 0
6762 && (eol_seen
& EOL_SEEN_CR
) != 0
6763 && (eol_seen
& EOL_SEEN_LF
) == 0)
6764 eol_seen
= EOL_SEEN_CRLF
;
6765 else if (eol_seen
!= EOL_SEEN_NONE
6766 && eol_seen
!= EOL_SEEN_LF
6767 && eol_seen
!= EOL_SEEN_CRLF
6768 && eol_seen
!= EOL_SEEN_CR
)
6769 eol_seen
= EOL_SEEN_LF
;
6770 if (eol_seen
!= EOL_SEEN_NONE
)
6771 eol_type
= adjust_coding_eol_type (coding
, eol_seen
);
6774 if (EQ (eol_type
, Qmac
))
6776 for (p
= pbeg
; p
< pend
; p
++)
6780 else if (EQ (eol_type
, Qdos
))
6784 if (NILP (coding
->dst_object
))
6786 /* Start deleting '\r' from the tail to minimize the memory
6788 for (p
= pend
- 2; p
>= pbeg
; p
--)
6791 memmove (p
, p
+ 1, pend
-- - p
- 1);
6797 ptrdiff_t pos_byte
= coding
->dst_pos_byte
;
6798 ptrdiff_t pos
= coding
->dst_pos
;
6799 ptrdiff_t pos_end
= pos
+ coding
->produced_char
- 1;
6801 while (pos
< pos_end
)
6803 p
= BYTE_POS_ADDR (pos_byte
);
6804 if (*p
== '\r' && p
[1] == '\n')
6806 del_range_2 (pos
, pos_byte
, pos
+ 1, pos_byte
+ 1, 0);
6811 if (coding
->dst_multibyte
)
6812 pos_byte
+= BYTES_BY_CHAR_HEAD (*p
);
6817 coding
->produced
-= n
;
6818 coding
->produced_char
-= n
;
6823 /* Return a translation table (or list of them) from coding system
6824 attribute vector ATTRS for encoding (if ENCODEP) or decoding (if
6828 get_translation_table (Lisp_Object attrs
, bool encodep
, int *max_lookup
)
6830 Lisp_Object standard
, translation_table
;
6833 if (NILP (Venable_character_translation
))
6840 translation_table
= CODING_ATTR_ENCODE_TBL (attrs
),
6841 standard
= Vstandard_translation_table_for_encode
;
6843 translation_table
= CODING_ATTR_DECODE_TBL (attrs
),
6844 standard
= Vstandard_translation_table_for_decode
;
6845 if (NILP (translation_table
))
6846 translation_table
= standard
;
6849 if (SYMBOLP (translation_table
))
6850 translation_table
= Fget (translation_table
, Qtranslation_table
);
6851 else if (CONSP (translation_table
))
6853 translation_table
= Fcopy_sequence (translation_table
);
6854 for (val
= translation_table
; CONSP (val
); val
= XCDR (val
))
6855 if (SYMBOLP (XCAR (val
)))
6856 XSETCAR (val
, Fget (XCAR (val
), Qtranslation_table
));
6858 if (CHAR_TABLE_P (standard
))
6860 if (CONSP (translation_table
))
6861 translation_table
= nconc2 (translation_table
,
6862 Fcons (standard
, Qnil
));
6864 translation_table
= Fcons (translation_table
,
6865 Fcons (standard
, Qnil
));
6872 if (CHAR_TABLE_P (translation_table
)
6873 && CHAR_TABLE_EXTRA_SLOTS (XCHAR_TABLE (translation_table
)) > 1)
6875 val
= XCHAR_TABLE (translation_table
)->extras
[1];
6876 if (NATNUMP (val
) && *max_lookup
< XFASTINT (val
))
6877 *max_lookup
= XFASTINT (val
);
6879 else if (CONSP (translation_table
))
6883 for (tail
= translation_table
; CONSP (tail
); tail
= XCDR (tail
))
6884 if (CHAR_TABLE_P (XCAR (tail
))
6885 && CHAR_TABLE_EXTRA_SLOTS (XCHAR_TABLE (XCAR (tail
))) > 1)
6887 Lisp_Object tailval
= XCHAR_TABLE (XCAR (tail
))->extras
[1];
6888 if (NATNUMP (tailval
) && *max_lookup
< XFASTINT (tailval
))
6889 *max_lookup
= XFASTINT (tailval
);
6893 return translation_table
;
6896 #define LOOKUP_TRANSLATION_TABLE(table, c, trans) \
6899 if (CHAR_TABLE_P (table)) \
6901 trans = CHAR_TABLE_REF (table, c); \
6902 if (CHARACTERP (trans)) \
6903 c = XFASTINT (trans), trans = Qnil; \
6905 else if (CONSP (table)) \
6909 for (tail = table; CONSP (tail); tail = XCDR (tail)) \
6910 if (CHAR_TABLE_P (XCAR (tail))) \
6912 trans = CHAR_TABLE_REF (XCAR (tail), c); \
6913 if (CHARACTERP (trans)) \
6914 c = XFASTINT (trans), trans = Qnil; \
6915 else if (! NILP (trans)) \
6922 /* Return a translation of character(s) at BUF according to TRANS.
6923 TRANS is TO-CHAR or ((FROM . TO) ...) where
6924 FROM = [FROM-CHAR ...], TO is TO-CHAR or [TO-CHAR ...].
6925 The return value is TO-CHAR or ([FROM-CHAR ...] . TO) if a
6926 translation is found, and Qnil if not found..
6927 If BUF is too short to lookup characters in FROM, return Qt. */
6930 get_translation (Lisp_Object trans
, int *buf
, int *buf_end
)
6933 if (INTEGERP (trans
))
6935 for (; CONSP (trans
); trans
= XCDR (trans
))
6937 Lisp_Object val
= XCAR (trans
);
6938 Lisp_Object from
= XCAR (val
);
6939 ptrdiff_t len
= ASIZE (from
);
6942 for (i
= 0; i
< len
; i
++)
6944 if (buf
+ i
== buf_end
)
6946 if (XINT (AREF (from
, i
)) != buf
[i
])
6957 produce_chars (struct coding_system
*coding
, Lisp_Object translation_table
,
6960 unsigned char *dst
= coding
->destination
+ coding
->produced
;
6961 unsigned char *dst_end
= coding
->destination
+ coding
->dst_bytes
;
6963 ptrdiff_t produced_chars
= 0;
6966 if (! coding
->chars_at_source
)
6968 /* Source characters are in coding->charbuf. */
6969 int *buf
= coding
->charbuf
;
6970 int *buf_end
= buf
+ coding
->charbuf_used
;
6972 if (EQ (coding
->src_object
, coding
->dst_object
))
6974 coding_set_source (coding
);
6975 dst_end
= ((unsigned char *) coding
->source
) + coding
->consumed
;
6978 while (buf
< buf_end
)
6985 ptrdiff_t from_nchars
= 1, to_nchars
= 1;
6986 Lisp_Object trans
= Qnil
;
6988 LOOKUP_TRANSLATION_TABLE (translation_table
, c
, trans
);
6991 trans
= get_translation (trans
, buf
, buf_end
);
6992 if (INTEGERP (trans
))
6994 else if (CONSP (trans
))
6996 from_nchars
= ASIZE (XCAR (trans
));
6997 trans
= XCDR (trans
);
6998 if (INTEGERP (trans
))
7002 to_nchars
= ASIZE (trans
);
7003 c
= XINT (AREF (trans
, 0));
7006 else if (EQ (trans
, Qt
) && ! last_block
)
7010 if ((dst_end
- dst
) / MAX_MULTIBYTE_LENGTH
< to_nchars
)
7012 if (((min (PTRDIFF_MAX
, SIZE_MAX
) - (buf_end
- buf
))
7013 / MAX_MULTIBYTE_LENGTH
)
7015 memory_full (SIZE_MAX
);
7016 dst
= alloc_destination (coding
,
7018 + MAX_MULTIBYTE_LENGTH
* to_nchars
,
7020 if (EQ (coding
->src_object
, coding
->dst_object
))
7022 coding_set_source (coding
);
7023 dst_end
= (((unsigned char *) coding
->source
)
7024 + coding
->consumed
);
7027 dst_end
= coding
->destination
+ coding
->dst_bytes
;
7030 for (i
= 0; i
< to_nchars
; i
++)
7033 c
= XINT (AREF (trans
, i
));
7034 if (coding
->dst_multibyte
7035 || ! CHAR_BYTE8_P (c
))
7036 CHAR_STRING_ADVANCE_NO_UNIFY (c
, dst
);
7038 *dst
++ = CHAR_TO_BYTE8 (c
);
7040 produced_chars
+= to_nchars
;
7044 /* This is an annotation datum. (-C) is the length. */
7047 carryover
= buf_end
- buf
;
7051 /* Source characters are at coding->source. */
7052 const unsigned char *src
= coding
->source
;
7053 const unsigned char *src_end
= src
+ coding
->consumed
;
7055 if (EQ (coding
->dst_object
, coding
->src_object
))
7056 dst_end
= (unsigned char *) src
;
7057 if (coding
->src_multibyte
!= coding
->dst_multibyte
)
7059 if (coding
->src_multibyte
)
7061 bool multibytep
= 1;
7062 ptrdiff_t consumed_chars
= 0;
7066 const unsigned char *src_base
= src
;
7072 if (EQ (coding
->src_object
, coding
->dst_object
))
7073 dst_end
= (unsigned char *) src
;
7076 ptrdiff_t offset
= src
- coding
->source
;
7078 dst
= alloc_destination (coding
, src_end
- src
+ 1,
7080 dst_end
= coding
->destination
+ coding
->dst_bytes
;
7081 coding_set_source (coding
);
7082 src
= coding
->source
+ offset
;
7083 src_end
= coding
->source
+ coding
->consumed
;
7084 if (EQ (coding
->src_object
, coding
->dst_object
))
7085 dst_end
= (unsigned char *) src
;
7095 while (src
< src_end
)
7097 bool multibytep
= 1;
7100 if (dst
>= dst_end
- 1)
7102 if (EQ (coding
->src_object
, coding
->dst_object
))
7103 dst_end
= (unsigned char *) src
;
7104 if (dst
>= dst_end
- 1)
7106 ptrdiff_t offset
= src
- coding
->source
;
7107 ptrdiff_t more_bytes
;
7109 if (EQ (coding
->src_object
, coding
->dst_object
))
7110 more_bytes
= ((src_end
- src
) / 2) + 2;
7112 more_bytes
= src_end
- src
+ 2;
7113 dst
= alloc_destination (coding
, more_bytes
, dst
);
7114 dst_end
= coding
->destination
+ coding
->dst_bytes
;
7115 coding_set_source (coding
);
7116 src
= coding
->source
+ offset
;
7117 src_end
= coding
->source
+ coding
->consumed
;
7118 if (EQ (coding
->src_object
, coding
->dst_object
))
7119 dst_end
= (unsigned char *) src
;
7127 if (!EQ (coding
->src_object
, coding
->dst_object
))
7129 ptrdiff_t require
= coding
->src_bytes
- coding
->dst_bytes
;
7133 ptrdiff_t offset
= src
- coding
->source
;
7135 dst
= alloc_destination (coding
, require
, dst
);
7136 coding_set_source (coding
);
7137 src
= coding
->source
+ offset
;
7138 src_end
= coding
->source
+ coding
->consumed
;
7141 produced_chars
= coding
->consumed_char
;
7142 while (src
< src_end
)
7147 produced
= dst
- (coding
->destination
+ coding
->produced
);
7148 if (BUFFERP (coding
->dst_object
) && produced_chars
> 0)
7149 insert_from_gap (produced_chars
, produced
, 0);
7150 coding
->produced
+= produced
;
7151 coding
->produced_char
+= produced_chars
;
7155 /* Compose text in CODING->object according to the annotation data at
7156 CHARBUF. CHARBUF is an array:
7157 [ -LENGTH ANNOTATION_MASK NCHARS NBYTES METHOD [ COMPONENTS... ] ]
7161 produce_composition (struct coding_system
*coding
, int *charbuf
, ptrdiff_t pos
)
7165 enum composition_method method
;
7166 Lisp_Object components
;
7168 len
= -charbuf
[0] - MAX_ANNOTATION_LENGTH
;
7169 to
= pos
+ charbuf
[2];
7170 method
= (enum composition_method
) (charbuf
[4]);
7172 if (method
== COMPOSITION_RELATIVE
)
7176 Lisp_Object args
[MAX_COMPOSITION_COMPONENTS
* 2 - 1];
7179 if (method
== COMPOSITION_WITH_RULE
)
7180 len
= charbuf
[2] * 3 - 2;
7181 charbuf
+= MAX_ANNOTATION_LENGTH
;
7182 /* charbuf = [ CHRA ... CHAR] or [ CHAR -2 RULE ... CHAR ] */
7183 for (i
= j
= 0; i
< len
&& charbuf
[i
] != -1; i
++, j
++)
7185 if (charbuf
[i
] >= 0)
7186 args
[j
] = make_number (charbuf
[i
]);
7190 args
[j
] = make_number (charbuf
[i
] % 0x100);
7193 components
= (i
== j
? Fstring (j
, args
) : Fvector (j
, args
));
7195 compose_text (pos
, to
, components
, Qnil
, coding
->dst_object
);
7199 /* Put `charset' property on text in CODING->object according to
7200 the annotation data at CHARBUF. CHARBUF is an array:
7201 [ -LENGTH ANNOTATION_MASK NCHARS CHARSET-ID ]
7205 produce_charset (struct coding_system
*coding
, int *charbuf
, ptrdiff_t pos
)
7207 ptrdiff_t from
= pos
- charbuf
[2];
7208 struct charset
*charset
= CHARSET_FROM_ID (charbuf
[3]);
7210 Fput_text_property (make_number (from
), make_number (pos
),
7211 Qcharset
, CHARSET_NAME (charset
),
7212 coding
->dst_object
);
7216 #define CHARBUF_SIZE 0x4000
7218 #define ALLOC_CONVERSION_WORK_AREA(coding) \
7220 coding->charbuf = SAFE_ALLOCA (CHARBUF_SIZE * sizeof (int)); \
7221 coding->charbuf_size = CHARBUF_SIZE; \
7226 produce_annotation (struct coding_system
*coding
, ptrdiff_t pos
)
7228 int *charbuf
= coding
->charbuf
;
7229 int *charbuf_end
= charbuf
+ coding
->charbuf_used
;
7231 if (NILP (coding
->dst_object
))
7234 while (charbuf
< charbuf_end
)
7240 int len
= -*charbuf
;
7245 case CODING_ANNOTATE_COMPOSITION_MASK
:
7246 produce_composition (coding
, charbuf
, pos
);
7248 case CODING_ANNOTATE_CHARSET_MASK
:
7249 produce_charset (coding
, charbuf
, pos
);
7257 /* Decode the data at CODING->src_object into CODING->dst_object.
7258 CODING->src_object is a buffer, a string, or nil.
7259 CODING->dst_object is a buffer.
7261 If CODING->src_object is a buffer, it must be the current buffer.
7262 In this case, if CODING->src_pos is positive, it is a position of
7263 the source text in the buffer, otherwise, the source text is in the
7264 gap area of the buffer, and CODING->src_pos specifies the offset of
7265 the text from GPT (which must be the same as PT). If this is the
7266 same buffer as CODING->dst_object, CODING->src_pos must be
7269 If CODING->src_object is a string, CODING->src_pos is an index to
7272 If CODING->src_object is nil, CODING->source must already point to
7273 the non-relocatable memory area. In this case, CODING->src_pos is
7274 an offset from CODING->source.
7276 The decoded data is inserted at the current point of the buffer
7281 decode_coding (struct coding_system
*coding
)
7284 Lisp_Object undo_list
;
7285 Lisp_Object translation_table
;
7286 struct ccl_spec cclspec
;
7292 if (BUFFERP (coding
->src_object
)
7293 && coding
->src_pos
> 0
7294 && coding
->src_pos
< GPT
7295 && coding
->src_pos
+ coding
->src_chars
> GPT
)
7296 move_gap_both (coding
->src_pos
, coding
->src_pos_byte
);
7299 if (BUFFERP (coding
->dst_object
))
7301 set_buffer_internal (XBUFFER (coding
->dst_object
));
7303 move_gap_both (PT
, PT_BYTE
);
7305 /* We must disable undo_list in order to record the whole insert
7306 transaction via record_insert at the end. But doing so also
7307 disables the recording of the first change to the undo_list.
7308 Therefore we check for first change here and record it via
7309 record_first_change if needed. */
7310 if (MODIFF
<= SAVE_MODIFF
)
7311 record_first_change ();
7313 undo_list
= BVAR (current_buffer
, undo_list
);
7314 bset_undo_list (current_buffer
, Qt
);
7317 coding
->consumed
= coding
->consumed_char
= 0;
7318 coding
->produced
= coding
->produced_char
= 0;
7319 coding
->chars_at_source
= 0;
7320 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
7323 ALLOC_CONVERSION_WORK_AREA (coding
);
7325 attrs
= CODING_ID_ATTRS (coding
->id
);
7326 translation_table
= get_translation_table (attrs
, 0, NULL
);
7329 if (coding
->decoder
== decode_coding_ccl
)
7331 coding
->spec
.ccl
= &cclspec
;
7332 setup_ccl_program (&cclspec
.ccl
, CODING_CCL_DECODER (coding
));
7336 ptrdiff_t pos
= coding
->dst_pos
+ coding
->produced_char
;
7338 coding_set_source (coding
);
7339 coding
->annotated
= 0;
7340 coding
->charbuf_used
= carryover
;
7341 (*(coding
->decoder
)) (coding
);
7342 coding_set_destination (coding
);
7343 carryover
= produce_chars (coding
, translation_table
, 0);
7344 if (coding
->annotated
)
7345 produce_annotation (coding
, pos
);
7346 for (i
= 0; i
< carryover
; i
++)
7348 = coding
->charbuf
[coding
->charbuf_used
- carryover
+ i
];
7350 while (coding
->result
== CODING_RESULT_INSUFFICIENT_DST
7351 || (coding
->consumed
< coding
->src_bytes
7352 && (coding
->result
== CODING_RESULT_SUCCESS
7353 || coding
->result
== CODING_RESULT_INVALID_SRC
)));
7357 coding_set_destination (coding
);
7358 coding
->charbuf_used
= carryover
;
7359 produce_chars (coding
, translation_table
, 1);
7362 coding
->carryover_bytes
= 0;
7363 if (coding
->consumed
< coding
->src_bytes
)
7365 int nbytes
= coding
->src_bytes
- coding
->consumed
;
7366 const unsigned char *src
;
7368 coding_set_source (coding
);
7369 coding_set_destination (coding
);
7370 src
= coding
->source
+ coding
->consumed
;
7372 if (coding
->mode
& CODING_MODE_LAST_BLOCK
)
7374 /* Flush out unprocessed data as binary chars. We are sure
7375 that the number of data is less than the size of
7377 coding
->charbuf_used
= 0;
7378 coding
->chars_at_source
= 0;
7380 while (nbytes
-- > 0)
7385 c
= BYTE8_TO_CHAR (c
);
7386 coding
->charbuf
[coding
->charbuf_used
++] = c
;
7388 produce_chars (coding
, Qnil
, 1);
7392 /* Record unprocessed bytes in coding->carryover. We are
7393 sure that the number of data is less than the size of
7394 coding->carryover. */
7395 unsigned char *p
= coding
->carryover
;
7397 if (nbytes
> sizeof coding
->carryover
)
7398 nbytes
= sizeof coding
->carryover
;
7399 coding
->carryover_bytes
= nbytes
;
7400 while (nbytes
-- > 0)
7403 coding
->consumed
= coding
->src_bytes
;
7406 if (! EQ (CODING_ID_EOL_TYPE (coding
->id
), Qunix
)
7407 && !inhibit_eol_conversion
)
7408 decode_eol (coding
);
7409 if (BUFFERP (coding
->dst_object
))
7411 bset_undo_list (current_buffer
, undo_list
);
7412 record_insert (coding
->dst_pos
, coding
->produced_char
);
7419 /* Extract an annotation datum from a composition starting at POS and
7420 ending before LIMIT of CODING->src_object (buffer or string), store
7421 the data in BUF, set *STOP to a starting position of the next
7422 composition (if any) or to LIMIT, and return the address of the
7423 next element of BUF.
7425 If such an annotation is not found, set *STOP to a starting
7426 position of a composition after POS (if any) or to LIMIT, and
7430 handle_composition_annotation (ptrdiff_t pos
, ptrdiff_t limit
,
7431 struct coding_system
*coding
, int *buf
,
7434 ptrdiff_t start
, end
;
7437 if (! find_composition (pos
, limit
, &start
, &end
, &prop
, coding
->src_object
)
7440 else if (start
> pos
)
7446 /* We found a composition. Store the corresponding
7447 annotation data in BUF. */
7449 enum composition_method method
= COMPOSITION_METHOD (prop
);
7450 int nchars
= COMPOSITION_LENGTH (prop
);
7452 ADD_COMPOSITION_DATA (buf
, nchars
, 0, method
);
7453 if (method
!= COMPOSITION_RELATIVE
)
7455 Lisp_Object components
;
7456 ptrdiff_t i
, len
, i_byte
;
7458 components
= COMPOSITION_COMPONENTS (prop
);
7459 if (VECTORP (components
))
7461 len
= ASIZE (components
);
7462 for (i
= 0; i
< len
; i
++)
7463 *buf
++ = XINT (AREF (components
, i
));
7465 else if (STRINGP (components
))
7467 len
= SCHARS (components
);
7471 FETCH_STRING_CHAR_ADVANCE (*buf
, components
, i
, i_byte
);
7475 else if (INTEGERP (components
))
7478 *buf
++ = XINT (components
);
7480 else if (CONSP (components
))
7482 for (len
= 0; CONSP (components
);
7483 len
++, components
= XCDR (components
))
7484 *buf
++ = XINT (XCAR (components
));
7492 if (find_composition (end
, limit
, &start
, &end
, &prop
,
7503 /* Extract an annotation datum from a text property `charset' at POS of
7504 CODING->src_object (buffer of string), store the data in BUF, set
7505 *STOP to the position where the value of `charset' property changes
7506 (limiting by LIMIT), and return the address of the next element of
7509 If the property value is nil, set *STOP to the position where the
7510 property value is non-nil (limiting by LIMIT), and return BUF. */
7513 handle_charset_annotation (ptrdiff_t pos
, ptrdiff_t limit
,
7514 struct coding_system
*coding
, int *buf
,
7517 Lisp_Object val
, next
;
7520 val
= Fget_text_property (make_number (pos
), Qcharset
, coding
->src_object
);
7521 if (! NILP (val
) && CHARSETP (val
))
7522 id
= XINT (CHARSET_SYMBOL_ID (val
));
7525 ADD_CHARSET_DATA (buf
, 0, id
);
7526 next
= Fnext_single_property_change (make_number (pos
), Qcharset
,
7528 make_number (limit
));
7529 *stop
= XINT (next
);
7535 consume_chars (struct coding_system
*coding
, Lisp_Object translation_table
,
7538 int *buf
= coding
->charbuf
;
7539 int *buf_end
= coding
->charbuf
+ coding
->charbuf_size
;
7540 const unsigned char *src
= coding
->source
+ coding
->consumed
;
7541 const unsigned char *src_end
= coding
->source
+ coding
->src_bytes
;
7542 ptrdiff_t pos
= coding
->src_pos
+ coding
->consumed_char
;
7543 ptrdiff_t end_pos
= coding
->src_pos
+ coding
->src_chars
;
7544 bool multibytep
= coding
->src_multibyte
;
7545 Lisp_Object eol_type
;
7547 ptrdiff_t stop
, stop_composition
, stop_charset
;
7548 int *lookup_buf
= NULL
;
7550 if (! NILP (translation_table
))
7551 lookup_buf
= alloca (sizeof (int) * max_lookup
);
7553 eol_type
= inhibit_eol_conversion
? Qunix
: CODING_ID_EOL_TYPE (coding
->id
);
7554 if (VECTORP (eol_type
))
7557 /* Note: composition handling is not yet implemented. */
7558 coding
->common_flags
&= ~CODING_ANNOTATE_COMPOSITION_MASK
;
7560 if (NILP (coding
->src_object
))
7561 stop
= stop_composition
= stop_charset
= end_pos
;
7564 if (coding
->common_flags
& CODING_ANNOTATE_COMPOSITION_MASK
)
7565 stop
= stop_composition
= pos
;
7567 stop
= stop_composition
= end_pos
;
7568 if (coding
->common_flags
& CODING_ANNOTATE_CHARSET_MASK
)
7569 stop
= stop_charset
= pos
;
7571 stop_charset
= end_pos
;
7574 /* Compensate for CRLF and conversion. */
7575 buf_end
-= 1 + MAX_ANNOTATION_LENGTH
;
7576 while (buf
< buf_end
)
7584 if (pos
== stop_composition
)
7585 buf
= handle_composition_annotation (pos
, end_pos
, coding
,
7586 buf
, &stop_composition
);
7587 if (pos
== stop_charset
)
7588 buf
= handle_charset_annotation (pos
, end_pos
, coding
,
7589 buf
, &stop_charset
);
7590 stop
= (stop_composition
< stop_charset
7591 ? stop_composition
: stop_charset
);
7598 if (coding
->encoder
== encode_coding_raw_text
7599 || coding
->encoder
== encode_coding_ccl
)
7601 else if ((bytes
= MULTIBYTE_LENGTH (src
, src_end
)) > 0)
7602 c
= STRING_CHAR_ADVANCE_NO_UNIFY (src
), pos
+= bytes
;
7604 c
= BYTE8_TO_CHAR (*src
), src
++, pos
++;
7607 c
= STRING_CHAR_ADVANCE_NO_UNIFY (src
), pos
++;
7608 if ((c
== '\r') && (coding
->mode
& CODING_MODE_SELECTIVE_DISPLAY
))
7610 if (! EQ (eol_type
, Qunix
))
7614 if (EQ (eol_type
, Qdos
))
7622 LOOKUP_TRANSLATION_TABLE (translation_table
, c
, trans
);
7627 ptrdiff_t from_nchars
= 1, to_nchars
= 1;
7628 int *lookup_buf_end
;
7629 const unsigned char *p
= src
;
7633 for (i
= 1; i
< max_lookup
&& p
< src_end
; i
++)
7634 lookup_buf
[i
] = STRING_CHAR_ADVANCE (p
);
7635 lookup_buf_end
= lookup_buf
+ i
;
7636 trans
= get_translation (trans
, lookup_buf
, lookup_buf_end
);
7637 if (INTEGERP (trans
))
7639 else if (CONSP (trans
))
7641 from_nchars
= ASIZE (XCAR (trans
));
7642 trans
= XCDR (trans
);
7643 if (INTEGERP (trans
))
7647 to_nchars
= ASIZE (trans
);
7648 if (buf_end
- buf
< to_nchars
)
7650 c
= XINT (AREF (trans
, 0));
7656 for (i
= 1; i
< to_nchars
; i
++)
7657 *buf
++ = XINT (AREF (trans
, i
));
7658 for (i
= 1; i
< from_nchars
; i
++, pos
++)
7659 src
+= MULTIBYTE_LENGTH_NO_CHECK (src
);
7663 coding
->consumed
= src
- coding
->source
;
7664 coding
->consumed_char
= pos
- coding
->src_pos
;
7665 coding
->charbuf_used
= buf
- coding
->charbuf
;
7666 coding
->chars_at_source
= 0;
7670 /* Encode the text at CODING->src_object into CODING->dst_object.
7671 CODING->src_object is a buffer or a string.
7672 CODING->dst_object is a buffer or nil.
7674 If CODING->src_object is a buffer, it must be the current buffer.
7675 In this case, if CODING->src_pos is positive, it is a position of
7676 the source text in the buffer, otherwise. the source text is in the
7677 gap area of the buffer, and coding->src_pos specifies the offset of
7678 the text from GPT (which must be the same as PT). If this is the
7679 same buffer as CODING->dst_object, CODING->src_pos must be
7680 negative and CODING should not have `pre-write-conversion'.
7682 If CODING->src_object is a string, CODING should not have
7683 `pre-write-conversion'.
7685 If CODING->dst_object is a buffer, the encoded data is inserted at
7686 the current point of that buffer.
7688 If CODING->dst_object is nil, the encoded data is placed at the
7689 memory area specified by CODING->destination. */
7692 encode_coding (struct coding_system
*coding
)
7695 Lisp_Object translation_table
;
7697 struct ccl_spec cclspec
;
7701 attrs
= CODING_ID_ATTRS (coding
->id
);
7702 if (coding
->encoder
== encode_coding_raw_text
)
7703 translation_table
= Qnil
, max_lookup
= 0;
7705 translation_table
= get_translation_table (attrs
, 1, &max_lookup
);
7707 if (BUFFERP (coding
->dst_object
))
7709 set_buffer_internal (XBUFFER (coding
->dst_object
));
7710 coding
->dst_multibyte
7711 = ! NILP (BVAR (current_buffer
, enable_multibyte_characters
));
7714 coding
->consumed
= coding
->consumed_char
= 0;
7715 coding
->produced
= coding
->produced_char
= 0;
7716 record_conversion_result (coding
, CODING_RESULT_SUCCESS
);
7719 ALLOC_CONVERSION_WORK_AREA (coding
);
7721 if (coding
->encoder
== encode_coding_ccl
)
7723 coding
->spec
.ccl
= &cclspec
;
7724 setup_ccl_program (&cclspec
.ccl
, CODING_CCL_ENCODER (coding
));
7727 coding_set_source (coding
);
7728 consume_chars (coding
, translation_table
, max_lookup
);
7729 coding_set_destination (coding
);
7730 (*(coding
->encoder
)) (coding
);
7731 } while (coding
->consumed_char
< coding
->src_chars
);
7733 if (BUFFERP (coding
->dst_object
) && coding
->produced_char
> 0)
7734 insert_from_gap (coding
->produced_char
, coding
->produced
, 0);
7740 /* Name (or base name) of work buffer for code conversion. */
7741 static Lisp_Object Vcode_conversion_workbuf_name
;
7743 /* A working buffer used by the top level conversion. Once it is
7744 created, it is never destroyed. It has the name
7745 Vcode_conversion_workbuf_name. The other working buffers are
7746 destroyed after the use is finished, and their names are modified
7747 versions of Vcode_conversion_workbuf_name. */
7748 static Lisp_Object Vcode_conversion_reused_workbuf
;
7750 /* True iff Vcode_conversion_reused_workbuf is already in use. */
7751 static bool reused_workbuf_in_use
;
7754 /* Return a working buffer of code conversion. MULTIBYTE specifies the
7755 multibyteness of returning buffer. */
7758 make_conversion_work_buffer (bool multibyte
)
7760 Lisp_Object name
, workbuf
;
7761 struct buffer
*current
;
7763 if (reused_workbuf_in_use
)
7765 name
= Fgenerate_new_buffer_name (Vcode_conversion_workbuf_name
, Qnil
);
7766 workbuf
= Fget_buffer_create (name
);
7770 reused_workbuf_in_use
= 1;
7771 if (NILP (Fbuffer_live_p (Vcode_conversion_reused_workbuf
)))
7772 Vcode_conversion_reused_workbuf
7773 = Fget_buffer_create (Vcode_conversion_workbuf_name
);
7774 workbuf
= Vcode_conversion_reused_workbuf
;
7776 current
= current_buffer
;
7777 set_buffer_internal (XBUFFER (workbuf
));
7778 /* We can't allow modification hooks to run in the work buffer. For
7779 instance, directory_files_internal assumes that file decoding
7780 doesn't compile new regexps. */
7781 Fset (Fmake_local_variable (Qinhibit_modification_hooks
), Qt
);
7783 bset_undo_list (current_buffer
, Qt
);
7784 bset_enable_multibyte_characters (current_buffer
, multibyte
? Qt
: Qnil
);
7785 set_buffer_internal (current
);
7791 code_conversion_restore (Lisp_Object arg
)
7793 Lisp_Object current
, workbuf
;
7794 struct gcpro gcpro1
;
7797 current
= XCAR (arg
);
7798 workbuf
= XCDR (arg
);
7799 if (! NILP (workbuf
))
7801 if (EQ (workbuf
, Vcode_conversion_reused_workbuf
))
7802 reused_workbuf_in_use
= 0;
7804 Fkill_buffer (workbuf
);
7806 set_buffer_internal (XBUFFER (current
));
7812 code_conversion_save (bool with_work_buf
, bool multibyte
)
7814 Lisp_Object workbuf
= Qnil
;
7817 workbuf
= make_conversion_work_buffer (multibyte
);
7818 record_unwind_protect (code_conversion_restore
,
7819 Fcons (Fcurrent_buffer (), workbuf
));
7824 decode_coding_gap (struct coding_system
*coding
,
7825 ptrdiff_t chars
, ptrdiff_t bytes
)
7827 ptrdiff_t count
= SPECPDL_INDEX ();
7830 coding
->src_object
= Fcurrent_buffer ();
7831 coding
->src_chars
= chars
;
7832 coding
->src_bytes
= bytes
;
7833 coding
->src_pos
= -chars
;
7834 coding
->src_pos_byte
= -bytes
;
7835 coding
->src_multibyte
= chars
< bytes
;
7836 coding
->dst_object
= coding
->src_object
;
7837 coding
->dst_pos
= PT
;
7838 coding
->dst_pos_byte
= PT_BYTE
;
7839 coding
->dst_multibyte
= ! NILP (BVAR (current_buffer
, enable_multibyte_characters
));
7841 coding
->head_ascii
= -1;
7842 coding
->detected_utf8_chars
= -1;
7843 coding
->eol_seen
= EOL_SEEN_NONE
;
7844 if (CODING_REQUIRE_DETECTION (coding
))
7845 detect_coding (coding
);
7846 attrs
= CODING_ID_ATTRS (coding
->id
);
7847 if (! disable_ascii_optimization
7848 && ! coding
->src_multibyte
7849 && ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
))
7850 && NILP (CODING_ATTR_POST_READ (attrs
))
7851 && NILP (get_translation_table (attrs
, 0, NULL
)))
7853 chars
= coding
->head_ascii
;
7855 chars
= check_ascii (coding
);
7858 /* There exists a non-ASCII byte. */
7859 if (EQ (CODING_ATTR_TYPE (attrs
), Qutf_8
))
7861 if (coding
->detected_utf8_chars
>= 0)
7862 chars
= coding
->detected_utf8_chars
;
7864 chars
= check_utf_8 (coding
);
7865 if (CODING_UTF_8_BOM (coding
) != utf_without_bom
7866 && coding
->head_ascii
== 0
7867 && coding
->source
[0] == UTF_8_BOM_1
7868 && coding
->source
[1] == UTF_8_BOM_2
7869 && coding
->source
[2] == UTF_8_BOM_3
)
7873 coding
->src_bytes
-= 3;
7881 Lisp_Object eol_type
;
7883 eol_type
= CODING_ID_EOL_TYPE (coding
->id
);
7884 if (VECTORP (eol_type
))
7886 if (coding
->eol_seen
!= EOL_SEEN_NONE
)
7887 eol_type
= adjust_coding_eol_type (coding
, coding
->eol_seen
);
7889 if (EQ (eol_type
, Qmac
))
7891 unsigned char *src_end
= GAP_END_ADDR
;
7892 unsigned char *src
= src_end
- coding
->src_bytes
;
7894 while (src
< src_end
)
7900 else if (EQ (eol_type
, Qdos
))
7902 unsigned char *src
= GAP_END_ADDR
;
7903 unsigned char *src_beg
= src
- coding
->src_bytes
;
7904 unsigned char *dst
= src
;
7907 while (src_beg
< src
)
7910 if (*src
== '\n' && src
> src_beg
&& src
[-1] == '\r')
7917 coding
->produced
= bytes
;
7918 coding
->produced_char
= chars
;
7919 insert_from_gap (chars
, bytes
, 1);
7923 code_conversion_save (0, 0);
7925 coding
->mode
|= CODING_MODE_LAST_BLOCK
;
7926 current_buffer
->text
->inhibit_shrinking
= 1;
7927 decode_coding (coding
);
7928 current_buffer
->text
->inhibit_shrinking
= 0;
7930 if (! NILP (CODING_ATTR_POST_READ (attrs
)))
7932 ptrdiff_t prev_Z
= Z
, prev_Z_BYTE
= Z_BYTE
;
7935 TEMP_SET_PT_BOTH (coding
->dst_pos
, coding
->dst_pos_byte
);
7936 val
= call1 (CODING_ATTR_POST_READ (attrs
),
7937 make_number (coding
->produced_char
));
7939 coding
->produced_char
+= Z
- prev_Z
;
7940 coding
->produced
+= Z_BYTE
- prev_Z_BYTE
;
7943 unbind_to (count
, Qnil
);
7947 /* Decode the text in the range FROM/FROM_BYTE and TO/TO_BYTE in
7948 SRC_OBJECT into DST_OBJECT by coding context CODING.
7950 SRC_OBJECT is a buffer, a string, or Qnil.
7952 If it is a buffer, the text is at point of the buffer. FROM and TO
7953 are positions in the buffer.
7955 If it is a string, the text is at the beginning of the string.
7956 FROM and TO are indices to the string.
7958 If it is nil, the text is at coding->source. FROM and TO are
7959 indices to coding->source.
7961 DST_OBJECT is a buffer, Qt, or Qnil.
7963 If it is a buffer, the decoded text is inserted at point of the
7964 buffer. If the buffer is the same as SRC_OBJECT, the source text
7967 If it is Qt, a string is made from the decoded text, and
7968 set in CODING->dst_object.
7970 If it is Qnil, the decoded text is stored at CODING->destination.
7971 The caller must allocate CODING->dst_bytes bytes at
7972 CODING->destination by xmalloc. If the decoded text is longer than
7973 CODING->dst_bytes, CODING->destination is relocated by xrealloc.
7977 decode_coding_object (struct coding_system
*coding
,
7978 Lisp_Object src_object
,
7979 ptrdiff_t from
, ptrdiff_t from_byte
,
7980 ptrdiff_t to
, ptrdiff_t to_byte
,
7981 Lisp_Object dst_object
)
7983 ptrdiff_t count
= SPECPDL_INDEX ();
7984 unsigned char *destination
IF_LINT (= NULL
);
7985 ptrdiff_t dst_bytes
IF_LINT (= 0);
7986 ptrdiff_t chars
= to
- from
;
7987 ptrdiff_t bytes
= to_byte
- from_byte
;
7989 ptrdiff_t saved_pt
= -1, saved_pt_byte
IF_LINT (= 0);
7990 bool need_marker_adjustment
= 0;
7991 Lisp_Object old_deactivate_mark
;
7993 old_deactivate_mark
= Vdeactivate_mark
;
7995 if (NILP (dst_object
))
7997 destination
= coding
->destination
;
7998 dst_bytes
= coding
->dst_bytes
;
8001 coding
->src_object
= src_object
;
8002 coding
->src_chars
= chars
;
8003 coding
->src_bytes
= bytes
;
8004 coding
->src_multibyte
= chars
< bytes
;
8006 if (STRINGP (src_object
))
8008 coding
->src_pos
= from
;
8009 coding
->src_pos_byte
= from_byte
;
8011 else if (BUFFERP (src_object
))
8013 set_buffer_internal (XBUFFER (src_object
));
8015 move_gap_both (from
, from_byte
);
8016 if (EQ (src_object
, dst_object
))
8018 struct Lisp_Marker
*tail
;
8020 for (tail
= BUF_MARKERS (current_buffer
); tail
; tail
= tail
->next
)
8022 tail
->need_adjustment
8023 = tail
->charpos
== (tail
->insertion_type
? from
: to
);
8024 need_marker_adjustment
|= tail
->need_adjustment
;
8026 saved_pt
= PT
, saved_pt_byte
= PT_BYTE
;
8027 TEMP_SET_PT_BOTH (from
, from_byte
);
8028 current_buffer
->text
->inhibit_shrinking
= 1;
8029 del_range_both (from
, from_byte
, to
, to_byte
, 1);
8030 coding
->src_pos
= -chars
;
8031 coding
->src_pos_byte
= -bytes
;
8035 coding
->src_pos
= from
;
8036 coding
->src_pos_byte
= from_byte
;
8040 if (CODING_REQUIRE_DETECTION (coding
))
8041 detect_coding (coding
);
8042 attrs
= CODING_ID_ATTRS (coding
->id
);
8044 if (EQ (dst_object
, Qt
)
8045 || (! NILP (CODING_ATTR_POST_READ (attrs
))
8046 && NILP (dst_object
)))
8048 coding
->dst_multibyte
= !CODING_FOR_UNIBYTE (coding
);
8049 coding
->dst_object
= code_conversion_save (1, coding
->dst_multibyte
);
8050 coding
->dst_pos
= BEG
;
8051 coding
->dst_pos_byte
= BEG_BYTE
;
8053 else if (BUFFERP (dst_object
))
8055 code_conversion_save (0, 0);
8056 coding
->dst_object
= dst_object
;
8057 coding
->dst_pos
= BUF_PT (XBUFFER (dst_object
));
8058 coding
->dst_pos_byte
= BUF_PT_BYTE (XBUFFER (dst_object
));
8059 coding
->dst_multibyte
8060 = ! NILP (BVAR (XBUFFER (dst_object
), enable_multibyte_characters
));
8064 code_conversion_save (0, 0);
8065 coding
->dst_object
= Qnil
;
8066 /* Most callers presume this will return a multibyte result, and they
8067 won't use `binary' or `raw-text' anyway, so let's not worry about
8068 CODING_FOR_UNIBYTE. */
8069 coding
->dst_multibyte
= 1;
8072 decode_coding (coding
);
8074 if (BUFFERP (coding
->dst_object
))
8075 set_buffer_internal (XBUFFER (coding
->dst_object
));
8077 if (! NILP (CODING_ATTR_POST_READ (attrs
)))
8079 struct gcpro gcpro1
, gcpro2
, gcpro3
, gcpro4
, gcpro5
;
8080 ptrdiff_t prev_Z
= Z
, prev_Z_BYTE
= Z_BYTE
;
8083 TEMP_SET_PT_BOTH (coding
->dst_pos
, coding
->dst_pos_byte
);
8084 GCPRO5 (coding
->src_object
, coding
->dst_object
, src_object
, dst_object
,
8085 old_deactivate_mark
);
8086 val
= safe_call1 (CODING_ATTR_POST_READ (attrs
),
8087 make_number (coding
->produced_char
));
8090 coding
->produced_char
+= Z
- prev_Z
;
8091 coding
->produced
+= Z_BYTE
- prev_Z_BYTE
;
8094 if (EQ (dst_object
, Qt
))
8096 coding
->dst_object
= Fbuffer_string ();
8098 else if (NILP (dst_object
) && BUFFERP (coding
->dst_object
))
8100 set_buffer_internal (XBUFFER (coding
->dst_object
));
8101 if (dst_bytes
< coding
->produced
)
8103 eassert (coding
->produced
> 0);
8104 destination
= xrealloc (destination
, coding
->produced
);
8105 if (BEGV
< GPT
&& GPT
< BEGV
+ coding
->produced_char
)
8106 move_gap_both (BEGV
, BEGV_BYTE
);
8107 memcpy (destination
, BEGV_ADDR
, coding
->produced
);
8108 coding
->destination
= destination
;
8114 /* This is the case of:
8115 (BUFFERP (src_object) && EQ (src_object, dst_object))
8116 As we have moved PT while replacing the original buffer
8117 contents, we must recover it now. */
8118 set_buffer_internal (XBUFFER (src_object
));
8119 current_buffer
->text
->inhibit_shrinking
= 0;
8120 if (saved_pt
< from
)
8121 TEMP_SET_PT_BOTH (saved_pt
, saved_pt_byte
);
8122 else if (saved_pt
< from
+ chars
)
8123 TEMP_SET_PT_BOTH (from
, from_byte
);
8124 else if (! NILP (BVAR (current_buffer
, enable_multibyte_characters
)))
8125 TEMP_SET_PT_BOTH (saved_pt
+ (coding
->produced_char
- chars
),
8126 saved_pt_byte
+ (coding
->produced
- bytes
));
8128 TEMP_SET_PT_BOTH (saved_pt
+ (coding
->produced
- bytes
),
8129 saved_pt_byte
+ (coding
->produced
- bytes
));
8131 if (need_marker_adjustment
)
8133 struct Lisp_Marker
*tail
;
8135 for (tail
= BUF_MARKERS (current_buffer
); tail
; tail
= tail
->next
)
8136 if (tail
->need_adjustment
)
8138 tail
->need_adjustment
= 0;
8139 if (tail
->insertion_type
)
8141 tail
->bytepos
= from_byte
;
8142 tail
->charpos
= from
;
8146 tail
->bytepos
= from_byte
+ coding
->produced
;
8148 = (NILP (BVAR (current_buffer
, enable_multibyte_characters
))
8149 ? tail
->bytepos
: from
+ coding
->produced_char
);
8155 Vdeactivate_mark
= old_deactivate_mark
;
8156 unbind_to (count
, coding
->dst_object
);
8161 encode_coding_object (struct coding_system
*coding
,
8162 Lisp_Object src_object
,
8163 ptrdiff_t from
, ptrdiff_t from_byte
,
8164 ptrdiff_t to
, ptrdiff_t to_byte
,
8165 Lisp_Object dst_object
)
8167 ptrdiff_t count
= SPECPDL_INDEX ();
8168 ptrdiff_t chars
= to
- from
;
8169 ptrdiff_t bytes
= to_byte
- from_byte
;
8171 ptrdiff_t saved_pt
= -1, saved_pt_byte
IF_LINT (= 0);
8172 bool need_marker_adjustment
= 0;
8173 bool kill_src_buffer
= 0;
8174 Lisp_Object old_deactivate_mark
;
8176 old_deactivate_mark
= Vdeactivate_mark
;
8178 coding
->src_object
= src_object
;
8179 coding
->src_chars
= chars
;
8180 coding
->src_bytes
= bytes
;
8181 coding
->src_multibyte
= chars
< bytes
;
8183 attrs
= CODING_ID_ATTRS (coding
->id
);
8185 if (EQ (src_object
, dst_object
))
8187 struct Lisp_Marker
*tail
;
8189 for (tail
= BUF_MARKERS (current_buffer
); tail
; tail
= tail
->next
)
8191 tail
->need_adjustment
8192 = tail
->charpos
== (tail
->insertion_type
? from
: to
);
8193 need_marker_adjustment
|= tail
->need_adjustment
;
8197 if (! NILP (CODING_ATTR_PRE_WRITE (attrs
)))
8199 coding
->src_object
= code_conversion_save (1, coding
->src_multibyte
);
8200 set_buffer_internal (XBUFFER (coding
->src_object
));
8201 if (STRINGP (src_object
))
8202 insert_from_string (src_object
, from
, from_byte
, chars
, bytes
, 0);
8203 else if (BUFFERP (src_object
))
8204 insert_from_buffer (XBUFFER (src_object
), from
, chars
, 0);
8206 insert_1_both ((char *) coding
->source
+ from
, chars
, bytes
, 0, 0, 0);
8208 if (EQ (src_object
, dst_object
))
8210 set_buffer_internal (XBUFFER (src_object
));
8211 saved_pt
= PT
, saved_pt_byte
= PT_BYTE
;
8212 del_range_both (from
, from_byte
, to
, to_byte
, 1);
8213 set_buffer_internal (XBUFFER (coding
->src_object
));
8217 struct gcpro gcpro1
, gcpro2
, gcpro3
, gcpro4
, gcpro5
;
8219 GCPRO5 (coding
->src_object
, coding
->dst_object
, src_object
, dst_object
,
8220 old_deactivate_mark
);
8221 safe_call2 (CODING_ATTR_PRE_WRITE (attrs
),
8222 make_number (BEG
), make_number (Z
));
8225 if (XBUFFER (coding
->src_object
) != current_buffer
)
8226 kill_src_buffer
= 1;
8227 coding
->src_object
= Fcurrent_buffer ();
8229 move_gap_both (BEG
, BEG_BYTE
);
8230 coding
->src_chars
= Z
- BEG
;
8231 coding
->src_bytes
= Z_BYTE
- BEG_BYTE
;
8232 coding
->src_pos
= BEG
;
8233 coding
->src_pos_byte
= BEG_BYTE
;
8234 coding
->src_multibyte
= Z
< Z_BYTE
;
8236 else if (STRINGP (src_object
))
8238 code_conversion_save (0, 0);
8239 coding
->src_pos
= from
;
8240 coding
->src_pos_byte
= from_byte
;
8242 else if (BUFFERP (src_object
))
8244 code_conversion_save (0, 0);
8245 set_buffer_internal (XBUFFER (src_object
));
8246 if (EQ (src_object
, dst_object
))
8248 saved_pt
= PT
, saved_pt_byte
= PT_BYTE
;
8249 coding
->src_object
= del_range_1 (from
, to
, 1, 1);
8250 coding
->src_pos
= 0;
8251 coding
->src_pos_byte
= 0;
8255 if (from
< GPT
&& to
>= GPT
)
8256 move_gap_both (from
, from_byte
);
8257 coding
->src_pos
= from
;
8258 coding
->src_pos_byte
= from_byte
;
8262 code_conversion_save (0, 0);
8264 if (BUFFERP (dst_object
))
8266 coding
->dst_object
= dst_object
;
8267 if (EQ (src_object
, dst_object
))
8269 coding
->dst_pos
= from
;
8270 coding
->dst_pos_byte
= from_byte
;
8274 struct buffer
*current
= current_buffer
;
8276 set_buffer_temp (XBUFFER (dst_object
));
8277 coding
->dst_pos
= PT
;
8278 coding
->dst_pos_byte
= PT_BYTE
;
8279 move_gap_both (coding
->dst_pos
, coding
->dst_pos_byte
);
8280 set_buffer_temp (current
);
8282 coding
->dst_multibyte
8283 = ! NILP (BVAR (XBUFFER (dst_object
), enable_multibyte_characters
));
8285 else if (EQ (dst_object
, Qt
))
8287 ptrdiff_t dst_bytes
= max (1, coding
->src_chars
);
8288 coding
->dst_object
= Qnil
;
8289 coding
->destination
= xmalloc (dst_bytes
);
8290 coding
->dst_bytes
= dst_bytes
;
8291 coding
->dst_multibyte
= 0;
8295 coding
->dst_object
= Qnil
;
8296 coding
->dst_multibyte
= 0;
8299 encode_coding (coding
);
8301 if (EQ (dst_object
, Qt
))
8303 if (BUFFERP (coding
->dst_object
))
8304 coding
->dst_object
= Fbuffer_string ();
8308 = make_unibyte_string ((char *) coding
->destination
,
8310 xfree (coding
->destination
);
8316 /* This is the case of:
8317 (BUFFERP (src_object) && EQ (src_object, dst_object))
8318 As we have moved PT while replacing the original buffer
8319 contents, we must recover it now. */
8320 set_buffer_internal (XBUFFER (src_object
));
8321 if (saved_pt
< from
)
8322 TEMP_SET_PT_BOTH (saved_pt
, saved_pt_byte
);
8323 else if (saved_pt
< from
+ chars
)
8324 TEMP_SET_PT_BOTH (from
, from_byte
);
8325 else if (! NILP (BVAR (current_buffer
, enable_multibyte_characters
)))
8326 TEMP_SET_PT_BOTH (saved_pt
+ (coding
->produced_char
- chars
),
8327 saved_pt_byte
+ (coding
->produced
- bytes
));
8329 TEMP_SET_PT_BOTH (saved_pt
+ (coding
->produced
- bytes
),
8330 saved_pt_byte
+ (coding
->produced
- bytes
));
8332 if (need_marker_adjustment
)
8334 struct Lisp_Marker
*tail
;
8336 for (tail
= BUF_MARKERS (current_buffer
); tail
; tail
= tail
->next
)
8337 if (tail
->need_adjustment
)
8339 tail
->need_adjustment
= 0;
8340 if (tail
->insertion_type
)
8342 tail
->bytepos
= from_byte
;
8343 tail
->charpos
= from
;
8347 tail
->bytepos
= from_byte
+ coding
->produced
;
8349 = (NILP (BVAR (current_buffer
, enable_multibyte_characters
))
8350 ? tail
->bytepos
: from
+ coding
->produced_char
);
8356 if (kill_src_buffer
)
8357 Fkill_buffer (coding
->src_object
);
8359 Vdeactivate_mark
= old_deactivate_mark
;
8360 unbind_to (count
, Qnil
);
8365 preferred_coding_system (void)
8367 int id
= coding_categories
[coding_priorities
[0]].id
;
8369 return CODING_ID_NAME (id
);
8372 #if defined (WINDOWSNT) || defined (CYGWIN)
8375 from_unicode (Lisp_Object str
)
8378 if (!STRING_MULTIBYTE (str
) &&
8381 str
= Fsubstring (str
, make_number (0), make_number (-1));
8384 return code_convert_string_norecord (str
, Qutf_16le
, 0);
8388 from_unicode_buffer (const wchar_t* wstr
)
8390 return from_unicode (
8391 make_unibyte_string (
8393 /* we get one of the two final 0 bytes for free. */
8394 1 + sizeof (wchar_t) * wcslen (wstr
)));
8398 to_unicode (Lisp_Object str
, Lisp_Object
*buf
)
8400 *buf
= code_convert_string_norecord (str
, Qutf_16le
, 1);
8401 /* We need to make another copy (in addition to the one made by
8402 code_convert_string_norecord) to ensure that the final string is
8403 _doubly_ zero terminated --- that is, that the string is
8404 terminated by two zero bytes and one utf-16le null character.
8405 Because strings are already terminated with a single zero byte,
8406 we just add one additional zero. */
8407 str
= make_uninit_string (SBYTES (*buf
) + 1);
8408 memcpy (SDATA (str
), SDATA (*buf
), SBYTES (*buf
));
8409 SDATA (str
) [SBYTES (*buf
)] = '\0';
8411 return WCSDATA (*buf
);
8414 #endif /* WINDOWSNT || CYGWIN */
8418 /*** 8. Emacs Lisp library functions ***/
8420 DEFUN ("coding-system-p", Fcoding_system_p
, Scoding_system_p
, 1, 1, 0,
8421 doc
: /* Return t if OBJECT is nil or a coding-system.
8422 See the documentation of `define-coding-system' for information
8423 about coding-system objects. */)
8424 (Lisp_Object object
)
8427 || CODING_SYSTEM_ID (object
) >= 0)
8429 if (! SYMBOLP (object
)
8430 || NILP (Fget (object
, Qcoding_system_define_form
)))
8435 DEFUN ("read-non-nil-coding-system", Fread_non_nil_coding_system
,
8436 Sread_non_nil_coding_system
, 1, 1, 0,
8437 doc
: /* Read a coding system from the minibuffer, prompting with string PROMPT. */)
8438 (Lisp_Object prompt
)
8443 val
= Fcompleting_read (prompt
, Vcoding_system_alist
, Qnil
,
8444 Qt
, Qnil
, Qcoding_system_history
, Qnil
, Qnil
);
8446 while (SCHARS (val
) == 0);
8447 return (Fintern (val
, Qnil
));
8450 DEFUN ("read-coding-system", Fread_coding_system
, Sread_coding_system
, 1, 2, 0,
8451 doc
: /* Read a coding system from the minibuffer, prompting with string PROMPT.
8452 If the user enters null input, return second argument DEFAULT-CODING-SYSTEM.
8453 Ignores case when completing coding systems (all Emacs coding systems
8454 are lower-case). */)
8455 (Lisp_Object prompt
, Lisp_Object default_coding_system
)
8458 ptrdiff_t count
= SPECPDL_INDEX ();
8460 if (SYMBOLP (default_coding_system
))
8461 default_coding_system
= SYMBOL_NAME (default_coding_system
);
8462 specbind (Qcompletion_ignore_case
, Qt
);
8463 val
= Fcompleting_read (prompt
, Vcoding_system_alist
, Qnil
,
8464 Qt
, Qnil
, Qcoding_system_history
,
8465 default_coding_system
, Qnil
);
8466 unbind_to (count
, Qnil
);
8467 return (SCHARS (val
) == 0 ? Qnil
: Fintern (val
, Qnil
));
8470 DEFUN ("check-coding-system", Fcheck_coding_system
, Scheck_coding_system
,
8472 doc
: /* Check validity of CODING-SYSTEM.
8473 If valid, return CODING-SYSTEM, else signal a `coding-system-error' error.
8474 It is valid if it is nil or a symbol defined as a coding system by the
8475 function `define-coding-system'. */)
8476 (Lisp_Object coding_system
)
8478 Lisp_Object define_form
;
8480 define_form
= Fget (coding_system
, Qcoding_system_define_form
);
8481 if (! NILP (define_form
))
8483 Fput (coding_system
, Qcoding_system_define_form
, Qnil
);
8484 safe_eval (define_form
);
8486 if (!NILP (Fcoding_system_p (coding_system
)))
8487 return coding_system
;
8488 xsignal1 (Qcoding_system_error
, coding_system
);
8492 /* Detect how the bytes at SRC of length SRC_BYTES are encoded. If
8493 HIGHEST, return the coding system of the highest
8494 priority among the detected coding systems. Otherwise return a
8495 list of detected coding systems sorted by their priorities. If
8496 MULTIBYTEP, it is assumed that the bytes are in correct
8497 multibyte form but contains only ASCII and eight-bit chars.
8498 Otherwise, the bytes are raw bytes.
8500 CODING-SYSTEM controls the detection as below:
8502 If it is nil, detect both text-format and eol-format. If the
8503 text-format part of CODING-SYSTEM is already specified
8504 (e.g. `iso-latin-1'), detect only eol-format. If the eol-format
8505 part of CODING-SYSTEM is already specified (e.g. `undecided-unix'),
8506 detect only text-format. */
8509 detect_coding_system (const unsigned char *src
,
8510 ptrdiff_t src_chars
, ptrdiff_t src_bytes
,
8511 bool highest
, bool multibytep
,
8512 Lisp_Object coding_system
)
8514 const unsigned char *src_end
= src
+ src_bytes
;
8515 Lisp_Object attrs
, eol_type
;
8516 Lisp_Object val
= Qnil
;
8517 struct coding_system coding
;
8519 struct coding_detection_info detect_info
;
8520 enum coding_category base_category
;
8521 bool null_byte_found
= 0, eight_bit_found
= 0;
8523 if (NILP (coding_system
))
8524 coding_system
= Qundecided
;
8525 setup_coding_system (coding_system
, &coding
);
8526 attrs
= CODING_ID_ATTRS (coding
.id
);
8527 eol_type
= CODING_ID_EOL_TYPE (coding
.id
);
8528 coding_system
= CODING_ATTR_BASE_NAME (attrs
);
8530 coding
.source
= src
;
8531 coding
.src_chars
= src_chars
;
8532 coding
.src_bytes
= src_bytes
;
8533 coding
.src_multibyte
= multibytep
;
8534 coding
.consumed
= 0;
8535 coding
.mode
|= CODING_MODE_LAST_BLOCK
;
8536 coding
.head_ascii
= 0;
8538 detect_info
.checked
= detect_info
.found
= detect_info
.rejected
= 0;
8540 /* At first, detect text-format if necessary. */
8541 base_category
= XINT (CODING_ATTR_CATEGORY (attrs
));
8542 if (base_category
== coding_category_undecided
)
8544 enum coding_category category
IF_LINT (= 0);
8545 struct coding_system
*this IF_LINT (= NULL
);
8547 int inhibit_nbd
/* null byte detection */
8548 = (coding
.spec
.undecided
.inhibit_nbd
> 0
8549 | (coding
.spec
.undecided
.inhibit_nbd
== 0
8550 & inhibit_null_byte_detection
));
8551 int inhibit_ied
/* iso escape detection */
8552 = (coding
.spec
.undecided
.inhibit_ied
> 0
8553 | (coding
.spec
.undecided
.inhibit_ied
== 0
8554 & inhibit_iso_escape_detection
));
8556 = coding
.spec
.undecided
.prefer_utf_8
;
8559 /* Skip all ASCII bytes except for a few ISO2022 controls. */
8560 for (; src
< src_end
; src
++)
8565 eight_bit_found
= 1;
8566 if (null_byte_found
)
8571 if ((c
== ISO_CODE_ESC
|| c
== ISO_CODE_SI
|| c
== ISO_CODE_SO
)
8573 && ! detect_info
.checked
)
8575 if (detect_coding_iso_2022 (&coding
, &detect_info
))
8577 /* We have scanned the whole data. */
8578 if (! (detect_info
.rejected
& CATEGORY_MASK_ISO_7_ELSE
))
8580 /* We didn't find an 8-bit code. We may
8581 have found a null-byte, but it's very
8582 rare that a binary file confirm to
8585 coding
.head_ascii
= src
- coding
.source
;
8587 detect_info
.rejected
|= ~CATEGORY_MASK_ISO_ESCAPE
;
8591 else if (! c
&& !inhibit_nbd
)
8593 null_byte_found
= 1;
8594 if (eight_bit_found
)
8597 if (! eight_bit_found
)
8598 coding
.head_ascii
++;
8600 else if (! eight_bit_found
)
8601 coding
.head_ascii
++;
8604 if (null_byte_found
|| eight_bit_found
8605 || coding
.head_ascii
< coding
.src_bytes
8606 || detect_info
.found
)
8608 if (coding
.head_ascii
== coding
.src_bytes
)
8609 /* As all bytes are 7-bit, we can ignore non-ISO-2022 codings. */
8610 for (i
= 0; i
< coding_category_raw_text
; i
++)
8612 category
= coding_priorities
[i
];
8613 this = coding_categories
+ category
;
8614 if (detect_info
.found
& (1 << category
))
8619 if (null_byte_found
)
8621 detect_info
.checked
|= ~CATEGORY_MASK_UTF_16
;
8622 detect_info
.rejected
|= ~CATEGORY_MASK_UTF_16
;
8624 else if (prefer_utf_8
8625 && detect_coding_utf_8 (&coding
, &detect_info
))
8627 detect_info
.checked
|= ~CATEGORY_MASK_UTF_8
;
8628 detect_info
.rejected
|= ~CATEGORY_MASK_UTF_8
;
8630 for (i
= 0; i
< coding_category_raw_text
; i
++)
8632 category
= coding_priorities
[i
];
8633 this = coding_categories
+ category
;
8637 /* No coding system of this category is defined. */
8638 detect_info
.rejected
|= (1 << category
);
8640 else if (category
>= coding_category_raw_text
)
8642 else if (detect_info
.checked
& (1 << category
))
8645 && (detect_info
.found
& (1 << category
)))
8648 else if ((*(this->detector
)) (&coding
, &detect_info
)
8650 && (detect_info
.found
& (1 << category
)))
8652 if (category
== coding_category_utf_16_auto
)
8654 if (detect_info
.found
& CATEGORY_MASK_UTF_16_LE
)
8655 category
= coding_category_utf_16_le
;
8657 category
= coding_category_utf_16_be
;
8665 if ((detect_info
.rejected
& CATEGORY_MASK_ANY
) == CATEGORY_MASK_ANY
8668 detect_info
.found
= CATEGORY_MASK_RAW_TEXT
;
8669 id
= CODING_SYSTEM_ID (Qno_conversion
);
8670 val
= Fcons (make_number (id
), Qnil
);
8672 else if (! detect_info
.rejected
&& ! detect_info
.found
)
8674 detect_info
.found
= CATEGORY_MASK_ANY
;
8675 id
= coding_categories
[coding_category_undecided
].id
;
8676 val
= Fcons (make_number (id
), Qnil
);
8680 if (detect_info
.found
)
8682 detect_info
.found
= 1 << category
;
8683 val
= Fcons (make_number (this->id
), Qnil
);
8686 for (i
= 0; i
< coding_category_raw_text
; i
++)
8687 if (! (detect_info
.rejected
& (1 << coding_priorities
[i
])))
8689 detect_info
.found
= 1 << coding_priorities
[i
];
8690 id
= coding_categories
[coding_priorities
[i
]].id
;
8691 val
= Fcons (make_number (id
), Qnil
);
8697 int mask
= detect_info
.rejected
| detect_info
.found
;
8700 for (i
= coding_category_raw_text
- 1; i
>= 0; i
--)
8702 category
= coding_priorities
[i
];
8703 if (! (mask
& (1 << category
)))
8705 found
|= 1 << category
;
8706 id
= coding_categories
[category
].id
;
8708 val
= Fcons (make_number (id
), val
);
8711 for (i
= coding_category_raw_text
- 1; i
>= 0; i
--)
8713 category
= coding_priorities
[i
];
8714 if (detect_info
.found
& (1 << category
))
8716 id
= coding_categories
[category
].id
;
8717 val
= Fcons (make_number (id
), val
);
8720 detect_info
.found
|= found
;
8723 else if (base_category
== coding_category_utf_8_auto
)
8725 if (detect_coding_utf_8 (&coding
, &detect_info
))
8727 struct coding_system
*this;
8729 if (detect_info
.found
& CATEGORY_MASK_UTF_8_SIG
)
8730 this = coding_categories
+ coding_category_utf_8_sig
;
8732 this = coding_categories
+ coding_category_utf_8_nosig
;
8733 val
= Fcons (make_number (this->id
), Qnil
);
8736 else if (base_category
== coding_category_utf_16_auto
)
8738 if (detect_coding_utf_16 (&coding
, &detect_info
))
8740 struct coding_system
*this;
8742 if (detect_info
.found
& CATEGORY_MASK_UTF_16_LE
)
8743 this = coding_categories
+ coding_category_utf_16_le
;
8744 else if (detect_info
.found
& CATEGORY_MASK_UTF_16_BE
)
8745 this = coding_categories
+ coding_category_utf_16_be
;
8746 else if (detect_info
.rejected
& CATEGORY_MASK_UTF_16_LE_NOSIG
)
8747 this = coding_categories
+ coding_category_utf_16_be_nosig
;
8749 this = coding_categories
+ coding_category_utf_16_le_nosig
;
8750 val
= Fcons (make_number (this->id
), Qnil
);
8755 detect_info
.found
= 1 << XINT (CODING_ATTR_CATEGORY (attrs
));
8756 val
= Fcons (make_number (coding
.id
), Qnil
);
8759 /* Then, detect eol-format if necessary. */
8761 int normal_eol
= -1, utf_16_be_eol
= -1, utf_16_le_eol
= -1;
8764 if (VECTORP (eol_type
))
8766 if (detect_info
.found
& ~CATEGORY_MASK_UTF_16
)
8768 if (null_byte_found
)
8769 normal_eol
= EOL_SEEN_LF
;
8771 normal_eol
= detect_eol (coding
.source
, src_bytes
,
8772 coding_category_raw_text
);
8774 if (detect_info
.found
& (CATEGORY_MASK_UTF_16_BE
8775 | CATEGORY_MASK_UTF_16_BE_NOSIG
))
8776 utf_16_be_eol
= detect_eol (coding
.source
, src_bytes
,
8777 coding_category_utf_16_be
);
8778 if (detect_info
.found
& (CATEGORY_MASK_UTF_16_LE
8779 | CATEGORY_MASK_UTF_16_LE_NOSIG
))
8780 utf_16_le_eol
= detect_eol (coding
.source
, src_bytes
,
8781 coding_category_utf_16_le
);
8785 if (EQ (eol_type
, Qunix
))
8786 normal_eol
= utf_16_be_eol
= utf_16_le_eol
= EOL_SEEN_LF
;
8787 else if (EQ (eol_type
, Qdos
))
8788 normal_eol
= utf_16_be_eol
= utf_16_le_eol
= EOL_SEEN_CRLF
;
8790 normal_eol
= utf_16_be_eol
= utf_16_le_eol
= EOL_SEEN_CR
;
8793 for (tail
= val
; CONSP (tail
); tail
= XCDR (tail
))
8795 enum coding_category category
;
8798 id
= XINT (XCAR (tail
));
8799 attrs
= CODING_ID_ATTRS (id
);
8800 category
= XINT (CODING_ATTR_CATEGORY (attrs
));
8801 eol_type
= CODING_ID_EOL_TYPE (id
);
8802 if (VECTORP (eol_type
))
8804 if (category
== coding_category_utf_16_be
8805 || category
== coding_category_utf_16_be_nosig
)
8806 this_eol
= utf_16_be_eol
;
8807 else if (category
== coding_category_utf_16_le
8808 || category
== coding_category_utf_16_le_nosig
)
8809 this_eol
= utf_16_le_eol
;
8811 this_eol
= normal_eol
;
8813 if (this_eol
== EOL_SEEN_LF
)
8814 XSETCAR (tail
, AREF (eol_type
, 0));
8815 else if (this_eol
== EOL_SEEN_CRLF
)
8816 XSETCAR (tail
, AREF (eol_type
, 1));
8817 else if (this_eol
== EOL_SEEN_CR
)
8818 XSETCAR (tail
, AREF (eol_type
, 2));
8820 XSETCAR (tail
, CODING_ID_NAME (id
));
8823 XSETCAR (tail
, CODING_ID_NAME (id
));
8827 return (highest
? (CONSP (val
) ? XCAR (val
) : Qnil
) : val
);
8831 DEFUN ("detect-coding-region", Fdetect_coding_region
, Sdetect_coding_region
,
8833 doc
: /* Detect coding system of the text in the region between START and END.
8834 Return a list of possible coding systems ordered by priority.
8835 The coding systems to try and their priorities follows what
8836 the function `coding-system-priority-list' (which see) returns.
8838 If only ASCII characters are found (except for such ISO-2022 control
8839 characters as ESC), it returns a list of single element `undecided'
8840 or its subsidiary coding system according to a detected end-of-line
8843 If optional argument HIGHEST is non-nil, return the coding system of
8844 highest priority. */)
8845 (Lisp_Object start
, Lisp_Object end
, Lisp_Object highest
)
8848 ptrdiff_t from_byte
, to_byte
;
8850 validate_region (&start
, &end
);
8851 from
= XINT (start
), to
= XINT (end
);
8852 from_byte
= CHAR_TO_BYTE (from
);
8853 to_byte
= CHAR_TO_BYTE (to
);
8855 if (from
< GPT
&& to
>= GPT
)
8856 move_gap_both (to
, to_byte
);
8858 return detect_coding_system (BYTE_POS_ADDR (from_byte
),
8859 to
- from
, to_byte
- from_byte
,
8861 !NILP (BVAR (current_buffer
8862 , enable_multibyte_characters
)),
8866 DEFUN ("detect-coding-string", Fdetect_coding_string
, Sdetect_coding_string
,
8868 doc
: /* Detect coding system of the text in STRING.
8869 Return a list of possible coding systems ordered by priority.
8870 The coding systems to try and their priorities follows what
8871 the function `coding-system-priority-list' (which see) returns.
8873 If only ASCII characters are found (except for such ISO-2022 control
8874 characters as ESC), it returns a list of single element `undecided'
8875 or its subsidiary coding system according to a detected end-of-line
8878 If optional argument HIGHEST is non-nil, return the coding system of
8879 highest priority. */)
8880 (Lisp_Object string
, Lisp_Object highest
)
8882 CHECK_STRING (string
);
8884 return detect_coding_system (SDATA (string
),
8885 SCHARS (string
), SBYTES (string
),
8886 !NILP (highest
), STRING_MULTIBYTE (string
),
8892 char_encodable_p (int c
, Lisp_Object attrs
)
8895 struct charset
*charset
;
8896 Lisp_Object translation_table
;
8898 translation_table
= CODING_ATTR_TRANS_TBL (attrs
);
8899 if (! NILP (translation_table
))
8900 c
= translate_char (translation_table
, c
);
8901 for (tail
= CODING_ATTR_CHARSET_LIST (attrs
);
8902 CONSP (tail
); tail
= XCDR (tail
))
8904 charset
= CHARSET_FROM_ID (XINT (XCAR (tail
)));
8905 if (CHAR_CHARSET_P (c
, charset
))
8908 return (! NILP (tail
));
8912 /* Return a list of coding systems that safely encode the text between
8913 START and END. If EXCLUDE is non-nil, it is a list of coding
8914 systems not to check. The returned list doesn't contain any such
8915 coding systems. In any case, if the text contains only ASCII or is
8916 unibyte, return t. */
8918 DEFUN ("find-coding-systems-region-internal",
8919 Ffind_coding_systems_region_internal
,
8920 Sfind_coding_systems_region_internal
, 2, 3, 0,
8921 doc
: /* Internal use only. */)
8922 (Lisp_Object start
, Lisp_Object end
, Lisp_Object exclude
)
8924 Lisp_Object coding_attrs_list
, safe_codings
;
8925 ptrdiff_t start_byte
, end_byte
;
8926 const unsigned char *p
, *pbeg
, *pend
;
8928 Lisp_Object tail
, elt
, work_table
;
8930 if (STRINGP (start
))
8932 if (!STRING_MULTIBYTE (start
)
8933 || SCHARS (start
) == SBYTES (start
))
8936 end_byte
= SBYTES (start
);
8940 CHECK_NUMBER_COERCE_MARKER (start
);
8941 CHECK_NUMBER_COERCE_MARKER (end
);
8942 if (XINT (start
) < BEG
|| XINT (end
) > Z
|| XINT (start
) > XINT (end
))
8943 args_out_of_range (start
, end
);
8944 if (NILP (BVAR (current_buffer
, enable_multibyte_characters
)))
8946 start_byte
= CHAR_TO_BYTE (XINT (start
));
8947 end_byte
= CHAR_TO_BYTE (XINT (end
));
8948 if (XINT (end
) - XINT (start
) == end_byte
- start_byte
)
8951 if (XINT (start
) < GPT
&& XINT (end
) > GPT
)
8953 if ((GPT
- XINT (start
)) < (XINT (end
) - GPT
))
8954 move_gap_both (XINT (start
), start_byte
);
8956 move_gap_both (XINT (end
), end_byte
);
8960 coding_attrs_list
= Qnil
;
8961 for (tail
= Vcoding_system_list
; CONSP (tail
); tail
= XCDR (tail
))
8963 || NILP (Fmemq (XCAR (tail
), exclude
)))
8967 attrs
= AREF (CODING_SYSTEM_SPEC (XCAR (tail
)), 0);
8968 if (EQ (XCAR (tail
), CODING_ATTR_BASE_NAME (attrs
)))
8970 ASET (attrs
, coding_attr_trans_tbl
,
8971 get_translation_table (attrs
, 1, NULL
));
8972 coding_attrs_list
= Fcons (attrs
, coding_attrs_list
);
8976 if (STRINGP (start
))
8977 p
= pbeg
= SDATA (start
);
8979 p
= pbeg
= BYTE_POS_ADDR (start_byte
);
8980 pend
= p
+ (end_byte
- start_byte
);
8982 while (p
< pend
&& ASCII_BYTE_P (*p
)) p
++;
8983 while (p
< pend
&& ASCII_BYTE_P (*(pend
- 1))) pend
--;
8985 work_table
= Fmake_char_table (Qnil
, Qnil
);
8988 if (ASCII_BYTE_P (*p
))
8992 c
= STRING_CHAR_ADVANCE (p
);
8993 if (!NILP (char_table_ref (work_table
, c
)))
8994 /* This character was already checked. Ignore it. */
8997 charset_map_loaded
= 0;
8998 for (tail
= coding_attrs_list
; CONSP (tail
);)
9003 else if (char_encodable_p (c
, elt
))
9005 else if (CONSP (XCDR (tail
)))
9007 XSETCAR (tail
, XCAR (XCDR (tail
)));
9008 XSETCDR (tail
, XCDR (XCDR (tail
)));
9012 XSETCAR (tail
, Qnil
);
9016 if (charset_map_loaded
)
9018 ptrdiff_t p_offset
= p
- pbeg
, pend_offset
= pend
- pbeg
;
9020 if (STRINGP (start
))
9021 pbeg
= SDATA (start
);
9023 pbeg
= BYTE_POS_ADDR (start_byte
);
9024 p
= pbeg
+ p_offset
;
9025 pend
= pbeg
+ pend_offset
;
9027 char_table_set (work_table
, c
, Qt
);
9031 safe_codings
= list2 (Qraw_text
, Qno_conversion
);
9032 for (tail
= coding_attrs_list
; CONSP (tail
); tail
= XCDR (tail
))
9033 if (! NILP (XCAR (tail
)))
9034 safe_codings
= Fcons (CODING_ATTR_BASE_NAME (XCAR (tail
)), safe_codings
);
9036 return safe_codings
;
9040 DEFUN ("unencodable-char-position", Funencodable_char_position
,
9041 Sunencodable_char_position
, 3, 5, 0,
9043 Return position of first un-encodable character in a region.
9044 START and END specify the region and CODING-SYSTEM specifies the
9045 encoding to check. Return nil if CODING-SYSTEM does encode the region.
9047 If optional 4th argument COUNT is non-nil, it specifies at most how
9048 many un-encodable characters to search. In this case, the value is a
9051 If optional 5th argument STRING is non-nil, it is a string to search
9052 for un-encodable characters. In that case, START and END are indexes
9054 (Lisp_Object start
, Lisp_Object end
, Lisp_Object coding_system
, Lisp_Object count
, Lisp_Object string
)
9057 struct coding_system coding
;
9058 Lisp_Object attrs
, charset_list
, translation_table
;
9059 Lisp_Object positions
;
9061 const unsigned char *p
, *stop
, *pend
;
9062 bool ascii_compatible
;
9064 setup_coding_system (Fcheck_coding_system (coding_system
), &coding
);
9065 attrs
= CODING_ID_ATTRS (coding
.id
);
9066 if (EQ (CODING_ATTR_TYPE (attrs
), Qraw_text
))
9068 ascii_compatible
= ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
));
9069 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
9070 translation_table
= get_translation_table (attrs
, 1, NULL
);
9074 validate_region (&start
, &end
);
9075 from
= XINT (start
);
9077 if (NILP (BVAR (current_buffer
, enable_multibyte_characters
))
9078 || (ascii_compatible
9079 && (to
- from
) == (CHAR_TO_BYTE (to
) - (CHAR_TO_BYTE (from
)))))
9081 p
= CHAR_POS_ADDR (from
);
9082 pend
= CHAR_POS_ADDR (to
);
9083 if (from
< GPT
&& to
>= GPT
)
9090 CHECK_STRING (string
);
9091 CHECK_NATNUM (start
);
9093 if (! (XINT (start
) <= XINT (end
) && XINT (end
) <= SCHARS (string
)))
9094 args_out_of_range_3 (string
, start
, end
);
9095 from
= XINT (start
);
9097 if (! STRING_MULTIBYTE (string
))
9099 p
= SDATA (string
) + string_char_to_byte (string
, from
);
9100 stop
= pend
= SDATA (string
) + string_char_to_byte (string
, to
);
9101 if (ascii_compatible
&& (to
- from
) == (pend
- p
))
9109 CHECK_NATNUM (count
);
9114 charset_map_loaded
= 0;
9119 if (ascii_compatible
)
9120 while (p
< stop
&& ASCII_BYTE_P (*p
))
9130 c
= STRING_CHAR_ADVANCE (p
);
9131 if (! (ASCII_CHAR_P (c
) && ascii_compatible
)
9132 && ! char_charset (translate_char (translation_table
, c
),
9133 charset_list
, NULL
))
9135 positions
= Fcons (make_number (from
), positions
);
9142 if (charset_map_loaded
&& NILP (string
))
9144 p
= CHAR_POS_ADDR (from
);
9145 pend
= CHAR_POS_ADDR (to
);
9146 if (from
< GPT
&& to
>= GPT
)
9150 charset_map_loaded
= 0;
9154 return (NILP (count
) ? Fcar (positions
) : Fnreverse (positions
));
9158 DEFUN ("check-coding-systems-region", Fcheck_coding_systems_region
,
9159 Scheck_coding_systems_region
, 3, 3, 0,
9160 doc
: /* Check if the region is encodable by coding systems.
9162 START and END are buffer positions specifying the region.
9163 CODING-SYSTEM-LIST is a list of coding systems to check.
9165 The value is an alist ((CODING-SYSTEM POS0 POS1 ...) ...), where
9166 CODING-SYSTEM is a member of CODING-SYSTEM-LIST and can't encode the
9167 whole region, POS0, POS1, ... are buffer positions where non-encodable
9168 characters are found.
9170 If all coding systems in CODING-SYSTEM-LIST can encode the region, the
9173 START may be a string. In that case, check if the string is
9174 encodable, and the value contains indices to the string instead of
9175 buffer positions. END is ignored.
9177 If the current buffer (or START if it is a string) is unibyte, the value
9179 (Lisp_Object start
, Lisp_Object end
, Lisp_Object coding_system_list
)
9182 ptrdiff_t start_byte
, end_byte
;
9184 const unsigned char *p
, *pbeg
, *pend
;
9186 Lisp_Object tail
, elt
, attrs
;
9188 if (STRINGP (start
))
9190 if (!STRING_MULTIBYTE (start
)
9191 || SCHARS (start
) == SBYTES (start
))
9194 end_byte
= SBYTES (start
);
9199 CHECK_NUMBER_COERCE_MARKER (start
);
9200 CHECK_NUMBER_COERCE_MARKER (end
);
9201 if (XINT (start
) < BEG
|| XINT (end
) > Z
|| XINT (start
) > XINT (end
))
9202 args_out_of_range (start
, end
);
9203 if (NILP (BVAR (current_buffer
, enable_multibyte_characters
)))
9205 start_byte
= CHAR_TO_BYTE (XINT (start
));
9206 end_byte
= CHAR_TO_BYTE (XINT (end
));
9207 if (XINT (end
) - XINT (start
) == end_byte
- start_byte
)
9210 if (XINT (start
) < GPT
&& XINT (end
) > GPT
)
9212 if ((GPT
- XINT (start
)) < (XINT (end
) - GPT
))
9213 move_gap_both (XINT (start
), start_byte
);
9215 move_gap_both (XINT (end
), end_byte
);
9221 for (tail
= coding_system_list
; CONSP (tail
); tail
= XCDR (tail
))
9224 attrs
= AREF (CODING_SYSTEM_SPEC (elt
), 0);
9225 ASET (attrs
, coding_attr_trans_tbl
,
9226 get_translation_table (attrs
, 1, NULL
));
9227 list
= Fcons (Fcons (elt
, Fcons (attrs
, Qnil
)), list
);
9230 if (STRINGP (start
))
9231 p
= pbeg
= SDATA (start
);
9233 p
= pbeg
= BYTE_POS_ADDR (start_byte
);
9234 pend
= p
+ (end_byte
- start_byte
);
9236 while (p
< pend
&& ASCII_BYTE_P (*p
)) p
++, pos
++;
9237 while (p
< pend
&& ASCII_BYTE_P (*(pend
- 1))) pend
--;
9241 if (ASCII_BYTE_P (*p
))
9245 c
= STRING_CHAR_ADVANCE (p
);
9247 charset_map_loaded
= 0;
9248 for (tail
= list
; CONSP (tail
); tail
= XCDR (tail
))
9250 elt
= XCDR (XCAR (tail
));
9251 if (! char_encodable_p (c
, XCAR (elt
)))
9252 XSETCDR (elt
, Fcons (make_number (pos
), XCDR (elt
)));
9254 if (charset_map_loaded
)
9256 ptrdiff_t p_offset
= p
- pbeg
, pend_offset
= pend
- pbeg
;
9258 if (STRINGP (start
))
9259 pbeg
= SDATA (start
);
9261 pbeg
= BYTE_POS_ADDR (start_byte
);
9262 p
= pbeg
+ p_offset
;
9263 pend
= pbeg
+ pend_offset
;
9271 for (; CONSP (tail
); tail
= XCDR (tail
))
9274 if (CONSP (XCDR (XCDR (elt
))))
9275 list
= Fcons (Fcons (XCAR (elt
), Fnreverse (XCDR (XCDR (elt
)))),
9284 code_convert_region (Lisp_Object start
, Lisp_Object end
,
9285 Lisp_Object coding_system
, Lisp_Object dst_object
,
9286 bool encodep
, bool norecord
)
9288 struct coding_system coding
;
9289 ptrdiff_t from
, from_byte
, to
, to_byte
;
9290 Lisp_Object src_object
;
9292 if (NILP (coding_system
))
9293 coding_system
= Qno_conversion
;
9295 CHECK_CODING_SYSTEM (coding_system
);
9296 src_object
= Fcurrent_buffer ();
9297 if (NILP (dst_object
))
9298 dst_object
= src_object
;
9299 else if (! EQ (dst_object
, Qt
))
9300 CHECK_BUFFER (dst_object
);
9302 validate_region (&start
, &end
);
9303 from
= XFASTINT (start
);
9304 from_byte
= CHAR_TO_BYTE (from
);
9305 to
= XFASTINT (end
);
9306 to_byte
= CHAR_TO_BYTE (to
);
9308 setup_coding_system (coding_system
, &coding
);
9309 coding
.mode
|= CODING_MODE_LAST_BLOCK
;
9312 encode_coding_object (&coding
, src_object
, from
, from_byte
, to
, to_byte
,
9315 decode_coding_object (&coding
, src_object
, from
, from_byte
, to
, to_byte
,
9318 Vlast_coding_system_used
= CODING_ID_NAME (coding
.id
);
9320 return (BUFFERP (dst_object
)
9321 ? make_number (coding
.produced_char
)
9322 : coding
.dst_object
);
9326 DEFUN ("decode-coding-region", Fdecode_coding_region
, Sdecode_coding_region
,
9327 3, 4, "r\nzCoding system: ",
9328 doc
: /* Decode the current region from the specified coding system.
9329 When called from a program, takes four arguments:
9330 START, END, CODING-SYSTEM, and DESTINATION.
9331 START and END are buffer positions.
9333 Optional 4th arguments DESTINATION specifies where the decoded text goes.
9334 If nil, the region between START and END is replaced by the decoded text.
9335 If buffer, the decoded text is inserted in that buffer after point (point
9337 In those cases, the length of the decoded text is returned.
9338 If DESTINATION is t, the decoded text is returned.
9340 This function sets `last-coding-system-used' to the precise coding system
9341 used (which may be different from CODING-SYSTEM if CODING-SYSTEM is
9342 not fully specified.) */)
9343 (Lisp_Object start
, Lisp_Object end
, Lisp_Object coding_system
, Lisp_Object destination
)
9345 return code_convert_region (start
, end
, coding_system
, destination
, 0, 0);
9348 DEFUN ("encode-coding-region", Fencode_coding_region
, Sencode_coding_region
,
9349 3, 4, "r\nzCoding system: ",
9350 doc
: /* Encode the current region by specified coding system.
9351 When called from a program, takes four arguments:
9352 START, END, CODING-SYSTEM and DESTINATION.
9353 START and END are buffer positions.
9355 Optional 4th arguments DESTINATION specifies where the encoded text goes.
9356 If nil, the region between START and END is replace by the encoded text.
9357 If buffer, the encoded text is inserted in that buffer after point (point
9359 In those cases, the length of the encoded text is returned.
9360 If DESTINATION is t, the encoded text is returned.
9362 This function sets `last-coding-system-used' to the precise coding system
9363 used (which may be different from CODING-SYSTEM if CODING-SYSTEM is
9364 not fully specified.) */)
9365 (Lisp_Object start
, Lisp_Object end
, Lisp_Object coding_system
, Lisp_Object destination
)
9367 return code_convert_region (start
, end
, coding_system
, destination
, 1, 0);
9371 code_convert_string (Lisp_Object string
, Lisp_Object coding_system
,
9372 Lisp_Object dst_object
, bool encodep
, bool nocopy
,
9375 struct coding_system coding
;
9376 ptrdiff_t chars
, bytes
;
9378 CHECK_STRING (string
);
9379 if (NILP (coding_system
))
9382 Vlast_coding_system_used
= Qno_conversion
;
9383 if (NILP (dst_object
))
9384 return (nocopy
? Fcopy_sequence (string
) : string
);
9387 if (NILP (coding_system
))
9388 coding_system
= Qno_conversion
;
9390 CHECK_CODING_SYSTEM (coding_system
);
9391 if (NILP (dst_object
))
9393 else if (! EQ (dst_object
, Qt
))
9394 CHECK_BUFFER (dst_object
);
9396 setup_coding_system (coding_system
, &coding
);
9397 coding
.mode
|= CODING_MODE_LAST_BLOCK
;
9398 chars
= SCHARS (string
);
9399 bytes
= SBYTES (string
);
9401 encode_coding_object (&coding
, string
, 0, 0, chars
, bytes
, dst_object
);
9403 decode_coding_object (&coding
, string
, 0, 0, chars
, bytes
, dst_object
);
9405 Vlast_coding_system_used
= CODING_ID_NAME (coding
.id
);
9407 return (BUFFERP (dst_object
)
9408 ? make_number (coding
.produced_char
)
9409 : coding
.dst_object
);
9413 /* Encode or decode STRING according to CODING_SYSTEM.
9414 Do not set Vlast_coding_system_used.
9416 This function is called only from macros DECODE_FILE and
9417 ENCODE_FILE, thus we ignore character composition. */
9420 code_convert_string_norecord (Lisp_Object string
, Lisp_Object coding_system
,
9423 return code_convert_string (string
, coding_system
, Qt
, encodep
, 0, 1);
9427 DEFUN ("decode-coding-string", Fdecode_coding_string
, Sdecode_coding_string
,
9429 doc
: /* Decode STRING which is encoded in CODING-SYSTEM, and return the result.
9431 Optional third arg NOCOPY non-nil means it is OK to return STRING itself
9432 if the decoding operation is trivial.
9434 Optional fourth arg BUFFER non-nil means that the decoded text is
9435 inserted in that buffer after point (point does not move). In this
9436 case, the return value is the length of the decoded text.
9438 This function sets `last-coding-system-used' to the precise coding system
9439 used (which may be different from CODING-SYSTEM if CODING-SYSTEM is
9440 not fully specified.) */)
9441 (Lisp_Object string
, Lisp_Object coding_system
, Lisp_Object nocopy
, Lisp_Object buffer
)
9443 return code_convert_string (string
, coding_system
, buffer
,
9444 0, ! NILP (nocopy
), 0);
9447 DEFUN ("encode-coding-string", Fencode_coding_string
, Sencode_coding_string
,
9449 doc
: /* Encode STRING to CODING-SYSTEM, and return the result.
9451 Optional third arg NOCOPY non-nil means it is OK to return STRING
9452 itself if the encoding operation is trivial.
9454 Optional fourth arg BUFFER non-nil means that the encoded text is
9455 inserted in that buffer after point (point does not move). In this
9456 case, the return value is the length of the encoded text.
9458 This function sets `last-coding-system-used' to the precise coding system
9459 used (which may be different from CODING-SYSTEM if CODING-SYSTEM is
9460 not fully specified.) */)
9461 (Lisp_Object string
, Lisp_Object coding_system
, Lisp_Object nocopy
, Lisp_Object buffer
)
9463 return code_convert_string (string
, coding_system
, buffer
,
9464 1, ! NILP (nocopy
), 0);
9468 DEFUN ("decode-sjis-char", Fdecode_sjis_char
, Sdecode_sjis_char
, 1, 1, 0,
9469 doc
: /* Decode a Japanese character which has CODE in shift_jis encoding.
9470 Return the corresponding character. */)
9473 Lisp_Object spec
, attrs
, val
;
9474 struct charset
*charset_roman
, *charset_kanji
, *charset_kana
, *charset
;
9478 CHECK_NATNUM (code
);
9479 ch
= XFASTINT (code
);
9480 CHECK_CODING_SYSTEM_GET_SPEC (Vsjis_coding_system
, spec
);
9481 attrs
= AREF (spec
, 0);
9483 if (ASCII_BYTE_P (ch
)
9484 && ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
)))
9487 val
= CODING_ATTR_CHARSET_LIST (attrs
);
9488 charset_roman
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
9489 charset_kana
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
9490 charset_kanji
= CHARSET_FROM_ID (XINT (XCAR (val
)));
9495 charset
= charset_roman
;
9497 else if (ch
>= 0xA0 && ch
< 0xDF)
9500 charset
= charset_kana
;
9504 EMACS_INT c1
= ch
>> 8;
9507 if (c1
< 0x81 || (c1
> 0x9F && c1
< 0xE0) || c1
> 0xEF
9508 || c2
< 0x40 || c2
== 0x7F || c2
> 0xFC)
9509 error ("Invalid code: %"pI
"d", ch
);
9512 charset
= charset_kanji
;
9514 c
= DECODE_CHAR (charset
, c
);
9516 error ("Invalid code: %"pI
"d", ch
);
9517 return make_number (c
);
9521 DEFUN ("encode-sjis-char", Fencode_sjis_char
, Sencode_sjis_char
, 1, 1, 0,
9522 doc
: /* Encode a Japanese character CH to shift_jis encoding.
9523 Return the corresponding code in SJIS. */)
9526 Lisp_Object spec
, attrs
, charset_list
;
9528 struct charset
*charset
;
9531 CHECK_CHARACTER (ch
);
9533 CHECK_CODING_SYSTEM_GET_SPEC (Vsjis_coding_system
, spec
);
9534 attrs
= AREF (spec
, 0);
9536 if (ASCII_CHAR_P (c
)
9537 && ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
)))
9540 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
9541 charset
= char_charset (c
, charset_list
, &code
);
9542 if (code
== CHARSET_INVALID_CODE (charset
))
9543 error ("Can't encode by shift_jis encoding: %c", c
);
9546 return make_number (code
);
9549 DEFUN ("decode-big5-char", Fdecode_big5_char
, Sdecode_big5_char
, 1, 1, 0,
9550 doc
: /* Decode a Big5 character which has CODE in BIG5 coding system.
9551 Return the corresponding character. */)
9554 Lisp_Object spec
, attrs
, val
;
9555 struct charset
*charset_roman
, *charset_big5
, *charset
;
9559 CHECK_NATNUM (code
);
9560 ch
= XFASTINT (code
);
9561 CHECK_CODING_SYSTEM_GET_SPEC (Vbig5_coding_system
, spec
);
9562 attrs
= AREF (spec
, 0);
9564 if (ASCII_BYTE_P (ch
)
9565 && ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
)))
9568 val
= CODING_ATTR_CHARSET_LIST (attrs
);
9569 charset_roman
= CHARSET_FROM_ID (XINT (XCAR (val
))), val
= XCDR (val
);
9570 charset_big5
= CHARSET_FROM_ID (XINT (XCAR (val
)));
9575 charset
= charset_roman
;
9579 EMACS_INT b1
= ch
>> 8;
9581 if (b1
< 0xA1 || b1
> 0xFE
9582 || b2
< 0x40 || (b2
> 0x7E && b2
< 0xA1) || b2
> 0xFE)
9583 error ("Invalid code: %"pI
"d", ch
);
9585 charset
= charset_big5
;
9587 c
= DECODE_CHAR (charset
, c
);
9589 error ("Invalid code: %"pI
"d", ch
);
9590 return make_number (c
);
9593 DEFUN ("encode-big5-char", Fencode_big5_char
, Sencode_big5_char
, 1, 1, 0,
9594 doc
: /* Encode the Big5 character CH to BIG5 coding system.
9595 Return the corresponding character code in Big5. */)
9598 Lisp_Object spec
, attrs
, charset_list
;
9599 struct charset
*charset
;
9603 CHECK_CHARACTER (ch
);
9605 CHECK_CODING_SYSTEM_GET_SPEC (Vbig5_coding_system
, spec
);
9606 attrs
= AREF (spec
, 0);
9607 if (ASCII_CHAR_P (c
)
9608 && ! NILP (CODING_ATTR_ASCII_COMPAT (attrs
)))
9611 charset_list
= CODING_ATTR_CHARSET_LIST (attrs
);
9612 charset
= char_charset (c
, charset_list
, &code
);
9613 if (code
== CHARSET_INVALID_CODE (charset
))
9614 error ("Can't encode by Big5 encoding: %c", c
);
9616 return make_number (code
);
9620 DEFUN ("set-terminal-coding-system-internal", Fset_terminal_coding_system_internal
,
9621 Sset_terminal_coding_system_internal
, 1, 2, 0,
9622 doc
: /* Internal use only. */)
9623 (Lisp_Object coding_system
, Lisp_Object terminal
)
9625 struct terminal
*term
= get_terminal (terminal
, 1);
9626 struct coding_system
*terminal_coding
= TERMINAL_TERMINAL_CODING (term
);
9627 CHECK_SYMBOL (coding_system
);
9628 setup_coding_system (Fcheck_coding_system (coding_system
), terminal_coding
);
9629 /* We had better not send unsafe characters to terminal. */
9630 terminal_coding
->mode
|= CODING_MODE_SAFE_ENCODING
;
9631 /* Character composition should be disabled. */
9632 terminal_coding
->common_flags
&= ~CODING_ANNOTATE_COMPOSITION_MASK
;
9633 terminal_coding
->src_multibyte
= 1;
9634 terminal_coding
->dst_multibyte
= 0;
9636 (term
, (terminal_coding
->common_flags
& CODING_REQUIRE_ENCODING_MASK
9637 ? coding_charset_list (terminal_coding
)
9638 : Fcons (make_number (charset_ascii
), Qnil
)));
9642 DEFUN ("set-safe-terminal-coding-system-internal",
9643 Fset_safe_terminal_coding_system_internal
,
9644 Sset_safe_terminal_coding_system_internal
, 1, 1, 0,
9645 doc
: /* Internal use only. */)
9646 (Lisp_Object coding_system
)
9648 CHECK_SYMBOL (coding_system
);
9649 setup_coding_system (Fcheck_coding_system (coding_system
),
9650 &safe_terminal_coding
);
9651 /* Character composition should be disabled. */
9652 safe_terminal_coding
.common_flags
&= ~CODING_ANNOTATE_COMPOSITION_MASK
;
9653 safe_terminal_coding
.src_multibyte
= 1;
9654 safe_terminal_coding
.dst_multibyte
= 0;
9658 DEFUN ("terminal-coding-system", Fterminal_coding_system
,
9659 Sterminal_coding_system
, 0, 1, 0,
9660 doc
: /* Return coding system specified for terminal output on the given terminal.
9661 TERMINAL may be a terminal object, a frame, or nil for the selected
9662 frame's terminal device. */)
9663 (Lisp_Object terminal
)
9665 struct coding_system
*terminal_coding
9666 = TERMINAL_TERMINAL_CODING (get_terminal (terminal
, 1));
9667 Lisp_Object coding_system
= CODING_ID_NAME (terminal_coding
->id
);
9669 /* For backward compatibility, return nil if it is `undecided'. */
9670 return (! EQ (coding_system
, Qundecided
) ? coding_system
: Qnil
);
9673 DEFUN ("set-keyboard-coding-system-internal", Fset_keyboard_coding_system_internal
,
9674 Sset_keyboard_coding_system_internal
, 1, 2, 0,
9675 doc
: /* Internal use only. */)
9676 (Lisp_Object coding_system
, Lisp_Object terminal
)
9678 struct terminal
*t
= get_terminal (terminal
, 1);
9679 CHECK_SYMBOL (coding_system
);
9680 if (NILP (coding_system
))
9681 coding_system
= Qno_conversion
;
9683 Fcheck_coding_system (coding_system
);
9684 setup_coding_system (coding_system
, TERMINAL_KEYBOARD_CODING (t
));
9685 /* Character composition should be disabled. */
9686 TERMINAL_KEYBOARD_CODING (t
)->common_flags
9687 &= ~CODING_ANNOTATE_COMPOSITION_MASK
;
9691 DEFUN ("keyboard-coding-system",
9692 Fkeyboard_coding_system
, Skeyboard_coding_system
, 0, 1, 0,
9693 doc
: /* Return coding system specified for decoding keyboard input. */)
9694 (Lisp_Object terminal
)
9696 return CODING_ID_NAME (TERMINAL_KEYBOARD_CODING
9697 (get_terminal (terminal
, 1))->id
);
9701 DEFUN ("find-operation-coding-system", Ffind_operation_coding_system
,
9702 Sfind_operation_coding_system
, 1, MANY
, 0,
9703 doc
: /* Choose a coding system for an operation based on the target name.
9704 The value names a pair of coding systems: (DECODING-SYSTEM . ENCODING-SYSTEM).
9705 DECODING-SYSTEM is the coding system to use for decoding
9706 \(in case OPERATION does decoding), and ENCODING-SYSTEM is the coding system
9707 for encoding (in case OPERATION does encoding).
9709 The first argument OPERATION specifies an I/O primitive:
9710 For file I/O, `insert-file-contents' or `write-region'.
9711 For process I/O, `call-process', `call-process-region', or `start-process'.
9712 For network I/O, `open-network-stream'.
9714 The remaining arguments should be the same arguments that were passed
9715 to the primitive. Depending on which primitive, one of those arguments
9716 is selected as the TARGET. For example, if OPERATION does file I/O,
9717 whichever argument specifies the file name is TARGET.
9719 TARGET has a meaning which depends on OPERATION:
9720 For file I/O, TARGET is a file name (except for the special case below).
9721 For process I/O, TARGET is a process name.
9722 For network I/O, TARGET is a service name or a port number.
9724 This function looks up what is specified for TARGET in
9725 `file-coding-system-alist', `process-coding-system-alist',
9726 or `network-coding-system-alist' depending on OPERATION.
9727 They may specify a coding system, a cons of coding systems,
9728 or a function symbol to call.
9729 In the last case, we call the function with one argument,
9730 which is a list of all the arguments given to this function.
9731 If the function can't decide a coding system, it can return
9732 `undecided' so that the normal code-detection is performed.
9734 If OPERATION is `insert-file-contents', the argument corresponding to
9735 TARGET may be a cons (FILENAME . BUFFER). In that case, FILENAME is a
9736 file name to look up, and BUFFER is a buffer that contains the file's
9737 contents (not yet decoded). If `file-coding-system-alist' specifies a
9738 function to call for FILENAME, that function should examine the
9739 contents of BUFFER instead of reading the file.
9741 usage: (find-operation-coding-system OPERATION ARGUMENTS...) */)
9742 (ptrdiff_t nargs
, Lisp_Object
*args
)
9744 Lisp_Object operation
, target_idx
, target
, val
;
9745 register Lisp_Object chain
;
9748 error ("Too few arguments");
9749 operation
= args
[0];
9750 if (!SYMBOLP (operation
)
9751 || (target_idx
= Fget (operation
, Qtarget_idx
), !NATNUMP (target_idx
)))
9752 error ("Invalid first argument");
9753 if (nargs
<= 1 + XFASTINT (target_idx
))
9754 error ("Too few arguments for operation `%s'",
9755 SDATA (SYMBOL_NAME (operation
)));
9756 target
= args
[XFASTINT (target_idx
) + 1];
9757 if (!(STRINGP (target
)
9758 || (EQ (operation
, Qinsert_file_contents
) && CONSP (target
)
9759 && STRINGP (XCAR (target
)) && BUFFERP (XCDR (target
)))
9760 || (EQ (operation
, Qopen_network_stream
) && INTEGERP (target
))))
9761 error ("Invalid argument %"pI
"d of operation `%s'",
9762 XFASTINT (target_idx
) + 1, SDATA (SYMBOL_NAME (operation
)));
9764 target
= XCAR (target
);
9766 chain
= ((EQ (operation
, Qinsert_file_contents
)
9767 || EQ (operation
, Qwrite_region
))
9768 ? Vfile_coding_system_alist
9769 : (EQ (operation
, Qopen_network_stream
)
9770 ? Vnetwork_coding_system_alist
9771 : Vprocess_coding_system_alist
));
9775 for (; CONSP (chain
); chain
= XCDR (chain
))
9781 && ((STRINGP (target
)
9782 && STRINGP (XCAR (elt
))
9783 && fast_string_match (XCAR (elt
), target
) >= 0)
9784 || (INTEGERP (target
) && EQ (target
, XCAR (elt
)))))
9787 /* Here, if VAL is both a valid coding system and a valid
9788 function symbol, we return VAL as a coding system. */
9791 if (! SYMBOLP (val
))
9793 if (! NILP (Fcoding_system_p (val
)))
9794 return Fcons (val
, val
);
9795 if (! NILP (Ffboundp (val
)))
9797 /* We use call1 rather than safe_call1
9798 so as to get bug reports about functions called here
9799 which don't handle the current interface. */
9800 val
= call1 (val
, Flist (nargs
, args
));
9803 if (SYMBOLP (val
) && ! NILP (Fcoding_system_p (val
)))
9804 return Fcons (val
, val
);
9812 DEFUN ("set-coding-system-priority", Fset_coding_system_priority
,
9813 Sset_coding_system_priority
, 0, MANY
, 0,
9814 doc
: /* Assign higher priority to the coding systems given as arguments.
9815 If multiple coding systems belong to the same category,
9816 all but the first one are ignored.
9818 usage: (set-coding-system-priority &rest coding-systems) */)
9819 (ptrdiff_t nargs
, Lisp_Object
*args
)
9822 bool changed
[coding_category_max
];
9823 enum coding_category priorities
[coding_category_max
];
9825 memset (changed
, 0, sizeof changed
);
9827 for (i
= j
= 0; i
< nargs
; i
++)
9829 enum coding_category category
;
9830 Lisp_Object spec
, attrs
;
9832 CHECK_CODING_SYSTEM_GET_SPEC (args
[i
], spec
);
9833 attrs
= AREF (spec
, 0);
9834 category
= XINT (CODING_ATTR_CATEGORY (attrs
));
9835 if (changed
[category
])
9836 /* Ignore this coding system because a coding system of the
9837 same category already had a higher priority. */
9839 changed
[category
] = 1;
9840 priorities
[j
++] = category
;
9841 if (coding_categories
[category
].id
>= 0
9842 && ! EQ (args
[i
], CODING_ID_NAME (coding_categories
[category
].id
)))
9843 setup_coding_system (args
[i
], &coding_categories
[category
]);
9844 Fset (AREF (Vcoding_category_table
, category
), args
[i
]);
9847 /* Now we have decided top J priorities. Reflect the order of the
9848 original priorities to the remaining priorities. */
9850 for (i
= j
, j
= 0; i
< coding_category_max
; i
++, j
++)
9852 while (j
< coding_category_max
9853 && changed
[coding_priorities
[j
]])
9855 if (j
== coding_category_max
)
9857 priorities
[i
] = coding_priorities
[j
];
9860 memcpy (coding_priorities
, priorities
, sizeof priorities
);
9862 /* Update `coding-category-list'. */
9863 Vcoding_category_list
= Qnil
;
9864 for (i
= coding_category_max
; i
-- > 0; )
9865 Vcoding_category_list
9866 = Fcons (AREF (Vcoding_category_table
, priorities
[i
]),
9867 Vcoding_category_list
);
9872 DEFUN ("coding-system-priority-list", Fcoding_system_priority_list
,
9873 Scoding_system_priority_list
, 0, 1, 0,
9874 doc
: /* Return a list of coding systems ordered by their priorities.
9875 The list contains a subset of coding systems; i.e. coding systems
9876 assigned to each coding category (see `coding-category-list').
9878 HIGHESTP non-nil means just return the highest priority one. */)
9879 (Lisp_Object highestp
)
9884 for (i
= 0, val
= Qnil
; i
< coding_category_max
; i
++)
9886 enum coding_category category
= coding_priorities
[i
];
9887 int id
= coding_categories
[category
].id
;
9892 attrs
= CODING_ID_ATTRS (id
);
9893 if (! NILP (highestp
))
9894 return CODING_ATTR_BASE_NAME (attrs
);
9895 val
= Fcons (CODING_ATTR_BASE_NAME (attrs
), val
);
9897 return Fnreverse (val
);
9900 static const char *const suffixes
[] = { "-unix", "-dos", "-mac" };
9903 make_subsidiaries (Lisp_Object base
)
9905 Lisp_Object subsidiaries
;
9906 ptrdiff_t base_name_len
= SBYTES (SYMBOL_NAME (base
));
9907 char *buf
= alloca (base_name_len
+ 6);
9910 memcpy (buf
, SDATA (SYMBOL_NAME (base
)), base_name_len
);
9911 subsidiaries
= make_uninit_vector (3);
9912 for (i
= 0; i
< 3; i
++)
9914 strcpy (buf
+ base_name_len
, suffixes
[i
]);
9915 ASET (subsidiaries
, i
, intern (buf
));
9917 return subsidiaries
;
9921 DEFUN ("define-coding-system-internal", Fdefine_coding_system_internal
,
9922 Sdefine_coding_system_internal
, coding_arg_max
, MANY
, 0,
9923 doc
: /* For internal use only.
9924 usage: (define-coding-system-internal ...) */)
9925 (ptrdiff_t nargs
, Lisp_Object
*args
)
9928 Lisp_Object spec_vec
; /* [ ATTRS ALIASE EOL_TYPE ] */
9929 Lisp_Object attrs
; /* Vector of attributes. */
9930 Lisp_Object eol_type
;
9931 Lisp_Object aliases
;
9932 Lisp_Object coding_type
, charset_list
, safe_charsets
;
9933 enum coding_category category
;
9934 Lisp_Object tail
, val
;
9935 int max_charset_id
= 0;
9938 if (nargs
< coding_arg_max
)
9941 attrs
= Fmake_vector (make_number (coding_attr_last_index
), Qnil
);
9943 name
= args
[coding_arg_name
];
9944 CHECK_SYMBOL (name
);
9945 ASET (attrs
, coding_attr_base_name
, name
);
9947 val
= args
[coding_arg_mnemonic
];
9948 if (! STRINGP (val
))
9949 CHECK_CHARACTER (val
);
9950 ASET (attrs
, coding_attr_mnemonic
, val
);
9952 coding_type
= args
[coding_arg_coding_type
];
9953 CHECK_SYMBOL (coding_type
);
9954 ASET (attrs
, coding_attr_type
, coding_type
);
9956 charset_list
= args
[coding_arg_charset_list
];
9957 if (SYMBOLP (charset_list
))
9959 if (EQ (charset_list
, Qiso_2022
))
9961 if (! EQ (coding_type
, Qiso_2022
))
9962 error ("Invalid charset-list");
9963 charset_list
= Viso_2022_charset_list
;
9965 else if (EQ (charset_list
, Qemacs_mule
))
9967 if (! EQ (coding_type
, Qemacs_mule
))
9968 error ("Invalid charset-list");
9969 charset_list
= Vemacs_mule_charset_list
;
9971 for (tail
= charset_list
; CONSP (tail
); tail
= XCDR (tail
))
9973 if (! RANGED_INTEGERP (0, XCAR (tail
), INT_MAX
- 1))
9974 error ("Invalid charset-list");
9975 if (max_charset_id
< XFASTINT (XCAR (tail
)))
9976 max_charset_id
= XFASTINT (XCAR (tail
));
9981 charset_list
= Fcopy_sequence (charset_list
);
9982 for (tail
= charset_list
; CONSP (tail
); tail
= XCDR (tail
))
9984 struct charset
*charset
;
9987 CHECK_CHARSET_GET_CHARSET (val
, charset
);
9988 if (EQ (coding_type
, Qiso_2022
)
9989 ? CHARSET_ISO_FINAL (charset
) < 0
9990 : EQ (coding_type
, Qemacs_mule
)
9991 ? CHARSET_EMACS_MULE_ID (charset
) < 0
9993 error ("Can't handle charset `%s'",
9994 SDATA (SYMBOL_NAME (CHARSET_NAME (charset
))));
9996 XSETCAR (tail
, make_number (charset
->id
));
9997 if (max_charset_id
< charset
->id
)
9998 max_charset_id
= charset
->id
;
10001 ASET (attrs
, coding_attr_charset_list
, charset_list
);
10003 safe_charsets
= make_uninit_string (max_charset_id
+ 1);
10004 memset (SDATA (safe_charsets
), 255, max_charset_id
+ 1);
10005 for (tail
= charset_list
; CONSP (tail
); tail
= XCDR (tail
))
10006 SSET (safe_charsets
, XFASTINT (XCAR (tail
)), 0);
10007 ASET (attrs
, coding_attr_safe_charsets
, safe_charsets
);
10009 ASET (attrs
, coding_attr_ascii_compat
, args
[coding_arg_ascii_compatible_p
]);
10011 val
= args
[coding_arg_decode_translation_table
];
10012 if (! CHAR_TABLE_P (val
) && ! CONSP (val
))
10013 CHECK_SYMBOL (val
);
10014 ASET (attrs
, coding_attr_decode_tbl
, val
);
10016 val
= args
[coding_arg_encode_translation_table
];
10017 if (! CHAR_TABLE_P (val
) && ! CONSP (val
))
10018 CHECK_SYMBOL (val
);
10019 ASET (attrs
, coding_attr_encode_tbl
, val
);
10021 val
= args
[coding_arg_post_read_conversion
];
10022 CHECK_SYMBOL (val
);
10023 ASET (attrs
, coding_attr_post_read
, val
);
10025 val
= args
[coding_arg_pre_write_conversion
];
10026 CHECK_SYMBOL (val
);
10027 ASET (attrs
, coding_attr_pre_write
, val
);
10029 val
= args
[coding_arg_default_char
];
10031 ASET (attrs
, coding_attr_default_char
, make_number (' '));
10034 CHECK_CHARACTER (val
);
10035 ASET (attrs
, coding_attr_default_char
, val
);
10038 val
= args
[coding_arg_for_unibyte
];
10039 ASET (attrs
, coding_attr_for_unibyte
, NILP (val
) ? Qnil
: Qt
);
10041 val
= args
[coding_arg_plist
];
10043 ASET (attrs
, coding_attr_plist
, val
);
10045 if (EQ (coding_type
, Qcharset
))
10047 /* Generate a lisp vector of 256 elements. Each element is nil,
10048 integer, or a list of charset IDs.
10050 If Nth element is nil, the byte code N is invalid in this
10053 If Nth element is a number NUM, N is the first byte of a
10054 charset whose ID is NUM.
10056 If Nth element is a list of charset IDs, N is the first byte
10057 of one of them. The list is sorted by dimensions of the
10058 charsets. A charset of smaller dimension comes first. */
10059 val
= Fmake_vector (make_number (256), Qnil
);
10061 for (tail
= charset_list
; CONSP (tail
); tail
= XCDR (tail
))
10063 struct charset
*charset
= CHARSET_FROM_ID (XFASTINT (XCAR (tail
)));
10064 int dim
= CHARSET_DIMENSION (charset
);
10065 int idx
= (dim
- 1) * 4;
10067 if (CHARSET_ASCII_COMPATIBLE_P (charset
))
10068 ASET (attrs
, coding_attr_ascii_compat
, Qt
);
10070 for (i
= charset
->code_space
[idx
];
10071 i
<= charset
->code_space
[idx
+ 1]; i
++)
10073 Lisp_Object tmp
, tmp2
;
10076 tmp
= AREF (val
, i
);
10079 else if (NUMBERP (tmp
))
10081 dim2
= CHARSET_DIMENSION (CHARSET_FROM_ID (XFASTINT (tmp
)));
10083 tmp
= Fcons (XCAR (tail
), Fcons (tmp
, Qnil
));
10085 tmp
= Fcons (tmp
, Fcons (XCAR (tail
), Qnil
));
10089 for (tmp2
= tmp
; CONSP (tmp2
); tmp2
= XCDR (tmp2
))
10091 dim2
= CHARSET_DIMENSION (CHARSET_FROM_ID (XFASTINT (XCAR (tmp2
))));
10096 tmp
= nconc2 (tmp
, Fcons (XCAR (tail
), Qnil
));
10099 XSETCDR (tmp2
, Fcons (XCAR (tmp2
), XCDR (tmp2
)));
10100 XSETCAR (tmp2
, XCAR (tail
));
10103 ASET (val
, i
, tmp
);
10106 ASET (attrs
, coding_attr_charset_valids
, val
);
10107 category
= coding_category_charset
;
10109 else if (EQ (coding_type
, Qccl
))
10111 Lisp_Object valids
;
10113 if (nargs
< coding_arg_ccl_max
)
10116 val
= args
[coding_arg_ccl_decoder
];
10117 CHECK_CCL_PROGRAM (val
);
10119 val
= Fcopy_sequence (val
);
10120 ASET (attrs
, coding_attr_ccl_decoder
, val
);
10122 val
= args
[coding_arg_ccl_encoder
];
10123 CHECK_CCL_PROGRAM (val
);
10125 val
= Fcopy_sequence (val
);
10126 ASET (attrs
, coding_attr_ccl_encoder
, val
);
10128 val
= args
[coding_arg_ccl_valids
];
10129 valids
= Fmake_string (make_number (256), make_number (0));
10130 for (tail
= val
; CONSP (tail
); tail
= XCDR (tail
))
10135 if (INTEGERP (val
))
10137 if (! (0 <= XINT (val
) && XINT (val
) <= 255))
10138 args_out_of_range_3 (val
, make_number (0), make_number (255));
10139 from
= to
= XINT (val
);
10144 CHECK_NATNUM_CAR (val
);
10145 CHECK_NUMBER_CDR (val
);
10146 if (XINT (XCAR (val
)) > 255)
10147 args_out_of_range_3 (XCAR (val
),
10148 make_number (0), make_number (255));
10149 from
= XINT (XCAR (val
));
10150 if (! (from
<= XINT (XCDR (val
)) && XINT (XCDR (val
)) <= 255))
10151 args_out_of_range_3 (XCDR (val
),
10152 XCAR (val
), make_number (255));
10153 to
= XINT (XCDR (val
));
10155 for (i
= from
; i
<= to
; i
++)
10156 SSET (valids
, i
, 1);
10158 ASET (attrs
, coding_attr_ccl_valids
, valids
);
10160 category
= coding_category_ccl
;
10162 else if (EQ (coding_type
, Qutf_16
))
10164 Lisp_Object bom
, endian
;
10166 ASET (attrs
, coding_attr_ascii_compat
, Qnil
);
10168 if (nargs
< coding_arg_utf16_max
)
10171 bom
= args
[coding_arg_utf16_bom
];
10172 if (! NILP (bom
) && ! EQ (bom
, Qt
))
10176 CHECK_CODING_SYSTEM (val
);
10178 CHECK_CODING_SYSTEM (val
);
10180 ASET (attrs
, coding_attr_utf_bom
, bom
);
10182 endian
= args
[coding_arg_utf16_endian
];
10183 CHECK_SYMBOL (endian
);
10186 else if (! EQ (endian
, Qbig
) && ! EQ (endian
, Qlittle
))
10187 error ("Invalid endian: %s", SDATA (SYMBOL_NAME (endian
)));
10188 ASET (attrs
, coding_attr_utf_16_endian
, endian
);
10190 category
= (CONSP (bom
)
10191 ? coding_category_utf_16_auto
10193 ? (EQ (endian
, Qbig
)
10194 ? coding_category_utf_16_be_nosig
10195 : coding_category_utf_16_le_nosig
)
10196 : (EQ (endian
, Qbig
)
10197 ? coding_category_utf_16_be
10198 : coding_category_utf_16_le
));
10200 else if (EQ (coding_type
, Qiso_2022
))
10202 Lisp_Object initial
, reg_usage
, request
, flags
;
10204 if (nargs
< coding_arg_iso2022_max
)
10207 initial
= Fcopy_sequence (args
[coding_arg_iso2022_initial
]);
10208 CHECK_VECTOR (initial
);
10209 for (i
= 0; i
< 4; i
++)
10211 val
= AREF (initial
, i
);
10214 struct charset
*charset
;
10216 CHECK_CHARSET_GET_CHARSET (val
, charset
);
10217 ASET (initial
, i
, make_number (CHARSET_ID (charset
)));
10218 if (i
== 0 && CHARSET_ASCII_COMPATIBLE_P (charset
))
10219 ASET (attrs
, coding_attr_ascii_compat
, Qt
);
10222 ASET (initial
, i
, make_number (-1));
10225 reg_usage
= args
[coding_arg_iso2022_reg_usage
];
10226 CHECK_CONS (reg_usage
);
10227 CHECK_NUMBER_CAR (reg_usage
);
10228 CHECK_NUMBER_CDR (reg_usage
);
10230 request
= Fcopy_sequence (args
[coding_arg_iso2022_request
]);
10231 for (tail
= request
; CONSP (tail
); tail
= XCDR (tail
))
10239 CHECK_CHARSET_GET_ID (tmp1
, id
);
10240 CHECK_NATNUM_CDR (val
);
10241 if (XINT (XCDR (val
)) >= 4)
10242 error ("Invalid graphic register number: %"pI
"d", XINT (XCDR (val
)));
10243 XSETCAR (val
, make_number (id
));
10246 flags
= args
[coding_arg_iso2022_flags
];
10247 CHECK_NATNUM (flags
);
10248 i
= XINT (flags
) & INT_MAX
;
10249 if (EQ (args
[coding_arg_charset_list
], Qiso_2022
))
10250 i
|= CODING_ISO_FLAG_FULL_SUPPORT
;
10251 flags
= make_number (i
);
10253 ASET (attrs
, coding_attr_iso_initial
, initial
);
10254 ASET (attrs
, coding_attr_iso_usage
, reg_usage
);
10255 ASET (attrs
, coding_attr_iso_request
, request
);
10256 ASET (attrs
, coding_attr_iso_flags
, flags
);
10257 setup_iso_safe_charsets (attrs
);
10259 if (i
& CODING_ISO_FLAG_SEVEN_BITS
)
10260 category
= ((i
& (CODING_ISO_FLAG_LOCKING_SHIFT
10261 | CODING_ISO_FLAG_SINGLE_SHIFT
))
10262 ? coding_category_iso_7_else
10263 : EQ (args
[coding_arg_charset_list
], Qiso_2022
)
10264 ? coding_category_iso_7
10265 : coding_category_iso_7_tight
);
10268 int id
= XINT (AREF (initial
, 1));
10270 category
= (((i
& CODING_ISO_FLAG_LOCKING_SHIFT
)
10271 || EQ (args
[coding_arg_charset_list
], Qiso_2022
)
10273 ? coding_category_iso_8_else
10274 : (CHARSET_DIMENSION (CHARSET_FROM_ID (id
)) == 1)
10275 ? coding_category_iso_8_1
10276 : coding_category_iso_8_2
);
10278 if (category
!= coding_category_iso_8_1
10279 && category
!= coding_category_iso_8_2
)
10280 ASET (attrs
, coding_attr_ascii_compat
, Qnil
);
10282 else if (EQ (coding_type
, Qemacs_mule
))
10284 if (EQ (args
[coding_arg_charset_list
], Qemacs_mule
))
10285 ASET (attrs
, coding_attr_emacs_mule_full
, Qt
);
10286 ASET (attrs
, coding_attr_ascii_compat
, Qt
);
10287 category
= coding_category_emacs_mule
;
10289 else if (EQ (coding_type
, Qshift_jis
))
10292 struct charset
*charset
;
10294 if (XINT (Flength (charset_list
)) != 3
10295 && XINT (Flength (charset_list
)) != 4)
10296 error ("There should be three or four charsets");
10298 charset
= CHARSET_FROM_ID (XINT (XCAR (charset_list
)));
10299 if (CHARSET_DIMENSION (charset
) != 1)
10300 error ("Dimension of charset %s is not one",
10301 SDATA (SYMBOL_NAME (CHARSET_NAME (charset
))));
10302 if (CHARSET_ASCII_COMPATIBLE_P (charset
))
10303 ASET (attrs
, coding_attr_ascii_compat
, Qt
);
10305 charset_list
= XCDR (charset_list
);
10306 charset
= CHARSET_FROM_ID (XINT (XCAR (charset_list
)));
10307 if (CHARSET_DIMENSION (charset
) != 1)
10308 error ("Dimension of charset %s is not one",
10309 SDATA (SYMBOL_NAME (CHARSET_NAME (charset
))));
10311 charset_list
= XCDR (charset_list
);
10312 charset
= CHARSET_FROM_ID (XINT (XCAR (charset_list
)));
10313 if (CHARSET_DIMENSION (charset
) != 2)
10314 error ("Dimension of charset %s is not two",
10315 SDATA (SYMBOL_NAME (CHARSET_NAME (charset
))));
10317 charset_list
= XCDR (charset_list
);
10318 if (! NILP (charset_list
))
10320 charset
= CHARSET_FROM_ID (XINT (XCAR (charset_list
)));
10321 if (CHARSET_DIMENSION (charset
) != 2)
10322 error ("Dimension of charset %s is not two",
10323 SDATA (SYMBOL_NAME (CHARSET_NAME (charset
))));
10326 category
= coding_category_sjis
;
10327 Vsjis_coding_system
= name
;
10329 else if (EQ (coding_type
, Qbig5
))
10331 struct charset
*charset
;
10333 if (XINT (Flength (charset_list
)) != 2)
10334 error ("There should be just two charsets");
10336 charset
= CHARSET_FROM_ID (XINT (XCAR (charset_list
)));
10337 if (CHARSET_DIMENSION (charset
) != 1)
10338 error ("Dimension of charset %s is not one",
10339 SDATA (SYMBOL_NAME (CHARSET_NAME (charset
))));
10340 if (CHARSET_ASCII_COMPATIBLE_P (charset
))
10341 ASET (attrs
, coding_attr_ascii_compat
, Qt
);
10343 charset_list
= XCDR (charset_list
);
10344 charset
= CHARSET_FROM_ID (XINT (XCAR (charset_list
)));
10345 if (CHARSET_DIMENSION (charset
) != 2)
10346 error ("Dimension of charset %s is not two",
10347 SDATA (SYMBOL_NAME (CHARSET_NAME (charset
))));
10349 category
= coding_category_big5
;
10350 Vbig5_coding_system
= name
;
10352 else if (EQ (coding_type
, Qraw_text
))
10354 category
= coding_category_raw_text
;
10355 ASET (attrs
, coding_attr_ascii_compat
, Qt
);
10357 else if (EQ (coding_type
, Qutf_8
))
10361 if (nargs
< coding_arg_utf8_max
)
10364 bom
= args
[coding_arg_utf8_bom
];
10365 if (! NILP (bom
) && ! EQ (bom
, Qt
))
10369 CHECK_CODING_SYSTEM (val
);
10371 CHECK_CODING_SYSTEM (val
);
10373 ASET (attrs
, coding_attr_utf_bom
, bom
);
10375 ASET (attrs
, coding_attr_ascii_compat
, Qt
);
10377 category
= (CONSP (bom
) ? coding_category_utf_8_auto
10378 : NILP (bom
) ? coding_category_utf_8_nosig
10379 : coding_category_utf_8_sig
);
10381 else if (EQ (coding_type
, Qundecided
))
10383 if (nargs
< coding_arg_undecided_max
)
10385 ASET (attrs
, coding_attr_undecided_inhibit_null_byte_detection
,
10386 args
[coding_arg_undecided_inhibit_null_byte_detection
]);
10387 ASET (attrs
, coding_attr_undecided_inhibit_iso_escape_detection
,
10388 args
[coding_arg_undecided_inhibit_iso_escape_detection
]);
10389 ASET (attrs
, coding_attr_undecided_prefer_utf_8
,
10390 args
[coding_arg_undecided_prefer_utf_8
]);
10391 category
= coding_category_undecided
;
10394 error ("Invalid coding system type: %s",
10395 SDATA (SYMBOL_NAME (coding_type
)));
10397 ASET (attrs
, coding_attr_category
, make_number (category
));
10398 ASET (attrs
, coding_attr_plist
,
10400 Fcons (AREF (Vcoding_category_table
, category
),
10401 CODING_ATTR_PLIST (attrs
))));
10402 ASET (attrs
, coding_attr_plist
,
10403 Fcons (QCascii_compatible_p
,
10404 Fcons (CODING_ATTR_ASCII_COMPAT (attrs
),
10405 CODING_ATTR_PLIST (attrs
))));
10407 eol_type
= args
[coding_arg_eol_type
];
10408 if (! NILP (eol_type
)
10409 && ! EQ (eol_type
, Qunix
)
10410 && ! EQ (eol_type
, Qdos
)
10411 && ! EQ (eol_type
, Qmac
))
10412 error ("Invalid eol-type");
10414 aliases
= Fcons (name
, Qnil
);
10416 if (NILP (eol_type
))
10418 eol_type
= make_subsidiaries (name
);
10419 for (i
= 0; i
< 3; i
++)
10421 Lisp_Object this_spec
, this_name
, this_aliases
, this_eol_type
;
10423 this_name
= AREF (eol_type
, i
);
10424 this_aliases
= Fcons (this_name
, Qnil
);
10425 this_eol_type
= (i
== 0 ? Qunix
: i
== 1 ? Qdos
: Qmac
);
10426 this_spec
= make_uninit_vector (3);
10427 ASET (this_spec
, 0, attrs
);
10428 ASET (this_spec
, 1, this_aliases
);
10429 ASET (this_spec
, 2, this_eol_type
);
10430 Fputhash (this_name
, this_spec
, Vcoding_system_hash_table
);
10431 Vcoding_system_list
= Fcons (this_name
, Vcoding_system_list
);
10432 val
= Fassoc (Fsymbol_name (this_name
), Vcoding_system_alist
);
10434 Vcoding_system_alist
10435 = Fcons (Fcons (Fsymbol_name (this_name
), Qnil
),
10436 Vcoding_system_alist
);
10440 spec_vec
= make_uninit_vector (3);
10441 ASET (spec_vec
, 0, attrs
);
10442 ASET (spec_vec
, 1, aliases
);
10443 ASET (spec_vec
, 2, eol_type
);
10445 Fputhash (name
, spec_vec
, Vcoding_system_hash_table
);
10446 Vcoding_system_list
= Fcons (name
, Vcoding_system_list
);
10447 val
= Fassoc (Fsymbol_name (name
), Vcoding_system_alist
);
10449 Vcoding_system_alist
= Fcons (Fcons (Fsymbol_name (name
), Qnil
),
10450 Vcoding_system_alist
);
10453 int id
= coding_categories
[category
].id
;
10455 if (id
< 0 || EQ (name
, CODING_ID_NAME (id
)))
10456 setup_coding_system (name
, &coding_categories
[category
]);
10462 return Fsignal (Qwrong_number_of_arguments
,
10463 Fcons (intern ("define-coding-system-internal"),
10464 make_number (nargs
)));
10468 DEFUN ("coding-system-put", Fcoding_system_put
, Scoding_system_put
,
10470 doc
: /* Change value in CODING-SYSTEM's property list PROP to VAL. */)
10471 (Lisp_Object coding_system
, Lisp_Object prop
, Lisp_Object val
)
10473 Lisp_Object spec
, attrs
;
10475 CHECK_CODING_SYSTEM_GET_SPEC (coding_system
, spec
);
10476 attrs
= AREF (spec
, 0);
10477 if (EQ (prop
, QCmnemonic
))
10479 if (! STRINGP (val
))
10480 CHECK_CHARACTER (val
);
10481 ASET (attrs
, coding_attr_mnemonic
, val
);
10483 else if (EQ (prop
, QCdefault_char
))
10486 val
= make_number (' ');
10488 CHECK_CHARACTER (val
);
10489 ASET (attrs
, coding_attr_default_char
, val
);
10491 else if (EQ (prop
, QCdecode_translation_table
))
10493 if (! CHAR_TABLE_P (val
) && ! CONSP (val
))
10494 CHECK_SYMBOL (val
);
10495 ASET (attrs
, coding_attr_decode_tbl
, val
);
10497 else if (EQ (prop
, QCencode_translation_table
))
10499 if (! CHAR_TABLE_P (val
) && ! CONSP (val
))
10500 CHECK_SYMBOL (val
);
10501 ASET (attrs
, coding_attr_encode_tbl
, val
);
10503 else if (EQ (prop
, QCpost_read_conversion
))
10505 CHECK_SYMBOL (val
);
10506 ASET (attrs
, coding_attr_post_read
, val
);
10508 else if (EQ (prop
, QCpre_write_conversion
))
10510 CHECK_SYMBOL (val
);
10511 ASET (attrs
, coding_attr_pre_write
, val
);
10513 else if (EQ (prop
, QCascii_compatible_p
))
10515 ASET (attrs
, coding_attr_ascii_compat
, val
);
10518 ASET (attrs
, coding_attr_plist
,
10519 Fplist_put (CODING_ATTR_PLIST (attrs
), prop
, val
));
10524 DEFUN ("define-coding-system-alias", Fdefine_coding_system_alias
,
10525 Sdefine_coding_system_alias
, 2, 2, 0,
10526 doc
: /* Define ALIAS as an alias for CODING-SYSTEM. */)
10527 (Lisp_Object alias
, Lisp_Object coding_system
)
10529 Lisp_Object spec
, aliases
, eol_type
, val
;
10531 CHECK_SYMBOL (alias
);
10532 CHECK_CODING_SYSTEM_GET_SPEC (coding_system
, spec
);
10533 aliases
= AREF (spec
, 1);
10534 /* ALIASES should be a list of length more than zero, and the first
10535 element is a base coding system. Append ALIAS at the tail of the
10537 while (!NILP (XCDR (aliases
)))
10538 aliases
= XCDR (aliases
);
10539 XSETCDR (aliases
, Fcons (alias
, Qnil
));
10541 eol_type
= AREF (spec
, 2);
10542 if (VECTORP (eol_type
))
10544 Lisp_Object subsidiaries
;
10547 subsidiaries
= make_subsidiaries (alias
);
10548 for (i
= 0; i
< 3; i
++)
10549 Fdefine_coding_system_alias (AREF (subsidiaries
, i
),
10550 AREF (eol_type
, i
));
10553 Fputhash (alias
, spec
, Vcoding_system_hash_table
);
10554 Vcoding_system_list
= Fcons (alias
, Vcoding_system_list
);
10555 val
= Fassoc (Fsymbol_name (alias
), Vcoding_system_alist
);
10557 Vcoding_system_alist
= Fcons (Fcons (Fsymbol_name (alias
), Qnil
),
10558 Vcoding_system_alist
);
10563 DEFUN ("coding-system-base", Fcoding_system_base
, Scoding_system_base
,
10565 doc
: /* Return the base of CODING-SYSTEM.
10566 Any alias or subsidiary coding system is not a base coding system. */)
10567 (Lisp_Object coding_system
)
10569 Lisp_Object spec
, attrs
;
10571 if (NILP (coding_system
))
10572 return (Qno_conversion
);
10573 CHECK_CODING_SYSTEM_GET_SPEC (coding_system
, spec
);
10574 attrs
= AREF (spec
, 0);
10575 return CODING_ATTR_BASE_NAME (attrs
);
10578 DEFUN ("coding-system-plist", Fcoding_system_plist
, Scoding_system_plist
,
10580 doc
: "Return the property list of CODING-SYSTEM.")
10581 (Lisp_Object coding_system
)
10583 Lisp_Object spec
, attrs
;
10585 if (NILP (coding_system
))
10586 coding_system
= Qno_conversion
;
10587 CHECK_CODING_SYSTEM_GET_SPEC (coding_system
, spec
);
10588 attrs
= AREF (spec
, 0);
10589 return CODING_ATTR_PLIST (attrs
);
10593 DEFUN ("coding-system-aliases", Fcoding_system_aliases
, Scoding_system_aliases
,
10595 doc
: /* Return the list of aliases of CODING-SYSTEM. */)
10596 (Lisp_Object coding_system
)
10600 if (NILP (coding_system
))
10601 coding_system
= Qno_conversion
;
10602 CHECK_CODING_SYSTEM_GET_SPEC (coding_system
, spec
);
10603 return AREF (spec
, 1);
10606 DEFUN ("coding-system-eol-type", Fcoding_system_eol_type
,
10607 Scoding_system_eol_type
, 1, 1, 0,
10608 doc
: /* Return eol-type of CODING-SYSTEM.
10609 An eol-type is an integer 0, 1, 2, or a vector of coding systems.
10611 Integer values 0, 1, and 2 indicate a format of end-of-line; LF, CRLF,
10612 and CR respectively.
10614 A vector value indicates that a format of end-of-line should be
10615 detected automatically. Nth element of the vector is the subsidiary
10616 coding system whose eol-type is N. */)
10617 (Lisp_Object coding_system
)
10619 Lisp_Object spec
, eol_type
;
10622 if (NILP (coding_system
))
10623 coding_system
= Qno_conversion
;
10624 if (! CODING_SYSTEM_P (coding_system
))
10626 spec
= CODING_SYSTEM_SPEC (coding_system
);
10627 eol_type
= AREF (spec
, 2);
10628 if (VECTORP (eol_type
))
10629 return Fcopy_sequence (eol_type
);
10630 n
= EQ (eol_type
, Qunix
) ? 0 : EQ (eol_type
, Qdos
) ? 1 : 2;
10631 return make_number (n
);
10637 /*** 9. Post-amble ***/
10640 init_coding_once (void)
10644 for (i
= 0; i
< coding_category_max
; i
++)
10646 coding_categories
[i
].id
= -1;
10647 coding_priorities
[i
] = i
;
10650 /* ISO2022 specific initialize routine. */
10651 for (i
= 0; i
< 0x20; i
++)
10652 iso_code_class
[i
] = ISO_control_0
;
10653 for (i
= 0x21; i
< 0x7F; i
++)
10654 iso_code_class
[i
] = ISO_graphic_plane_0
;
10655 for (i
= 0x80; i
< 0xA0; i
++)
10656 iso_code_class
[i
] = ISO_control_1
;
10657 for (i
= 0xA1; i
< 0xFF; i
++)
10658 iso_code_class
[i
] = ISO_graphic_plane_1
;
10659 iso_code_class
[0x20] = iso_code_class
[0x7F] = ISO_0x20_or_0x7F
;
10660 iso_code_class
[0xA0] = iso_code_class
[0xFF] = ISO_0xA0_or_0xFF
;
10661 iso_code_class
[ISO_CODE_SO
] = ISO_shift_out
;
10662 iso_code_class
[ISO_CODE_SI
] = ISO_shift_in
;
10663 iso_code_class
[ISO_CODE_SS2_7
] = ISO_single_shift_2_7
;
10664 iso_code_class
[ISO_CODE_ESC
] = ISO_escape
;
10665 iso_code_class
[ISO_CODE_SS2
] = ISO_single_shift_2
;
10666 iso_code_class
[ISO_CODE_SS3
] = ISO_single_shift_3
;
10667 iso_code_class
[ISO_CODE_CSI
] = ISO_control_sequence_introducer
;
10669 for (i
= 0; i
< 256; i
++)
10671 emacs_mule_bytes
[i
] = 1;
10673 emacs_mule_bytes
[EMACS_MULE_LEADING_CODE_PRIVATE_11
] = 3;
10674 emacs_mule_bytes
[EMACS_MULE_LEADING_CODE_PRIVATE_12
] = 3;
10675 emacs_mule_bytes
[EMACS_MULE_LEADING_CODE_PRIVATE_21
] = 4;
10676 emacs_mule_bytes
[EMACS_MULE_LEADING_CODE_PRIVATE_22
] = 4;
10682 syms_of_coding (void)
10684 staticpro (&Vcoding_system_hash_table
);
10686 Lisp_Object args
[2];
10689 Vcoding_system_hash_table
= Fmake_hash_table (2, args
);
10692 staticpro (&Vsjis_coding_system
);
10693 Vsjis_coding_system
= Qnil
;
10695 staticpro (&Vbig5_coding_system
);
10696 Vbig5_coding_system
= Qnil
;
10698 staticpro (&Vcode_conversion_reused_workbuf
);
10699 Vcode_conversion_reused_workbuf
= Qnil
;
10701 staticpro (&Vcode_conversion_workbuf_name
);
10702 Vcode_conversion_workbuf_name
= build_pure_c_string (" *code-conversion-work*");
10704 reused_workbuf_in_use
= 0;
10706 DEFSYM (Qcharset
, "charset");
10707 DEFSYM (Qtarget_idx
, "target-idx");
10708 DEFSYM (Qcoding_system_history
, "coding-system-history");
10709 Fset (Qcoding_system_history
, Qnil
);
10711 /* Target FILENAME is the first argument. */
10712 Fput (Qinsert_file_contents
, Qtarget_idx
, make_number (0));
10713 /* Target FILENAME is the third argument. */
10714 Fput (Qwrite_region
, Qtarget_idx
, make_number (2));
10716 DEFSYM (Qcall_process
, "call-process");
10717 /* Target PROGRAM is the first argument. */
10718 Fput (Qcall_process
, Qtarget_idx
, make_number (0));
10720 DEFSYM (Qcall_process_region
, "call-process-region");
10721 /* Target PROGRAM is the third argument. */
10722 Fput (Qcall_process_region
, Qtarget_idx
, make_number (2));
10724 DEFSYM (Qstart_process
, "start-process");
10725 /* Target PROGRAM is the third argument. */
10726 Fput (Qstart_process
, Qtarget_idx
, make_number (2));
10728 DEFSYM (Qopen_network_stream
, "open-network-stream");
10729 /* Target SERVICE is the fourth argument. */
10730 Fput (Qopen_network_stream
, Qtarget_idx
, make_number (3));
10732 DEFSYM (Qcoding_system
, "coding-system");
10733 DEFSYM (Qcoding_aliases
, "coding-aliases");
10735 DEFSYM (Qeol_type
, "eol-type");
10736 DEFSYM (Qunix
, "unix");
10737 DEFSYM (Qdos
, "dos");
10738 DEFSYM (Qmac
, "mac");
10740 DEFSYM (Qbuffer_file_coding_system
, "buffer-file-coding-system");
10741 DEFSYM (Qpost_read_conversion
, "post-read-conversion");
10742 DEFSYM (Qpre_write_conversion
, "pre-write-conversion");
10743 DEFSYM (Qdefault_char
, "default-char");
10744 DEFSYM (Qundecided
, "undecided");
10745 DEFSYM (Qno_conversion
, "no-conversion");
10746 DEFSYM (Qraw_text
, "raw-text");
10748 DEFSYM (Qiso_2022
, "iso-2022");
10750 DEFSYM (Qutf_8
, "utf-8");
10751 DEFSYM (Qutf_8_emacs
, "utf-8-emacs");
10753 #if defined (WINDOWSNT) || defined (CYGWIN)
10754 /* No, not utf-16-le: that one has a BOM. */
10755 DEFSYM (Qutf_16le
, "utf-16le");
10758 DEFSYM (Qutf_16
, "utf-16");
10759 DEFSYM (Qbig
, "big");
10760 DEFSYM (Qlittle
, "little");
10762 DEFSYM (Qshift_jis
, "shift-jis");
10763 DEFSYM (Qbig5
, "big5");
10765 DEFSYM (Qcoding_system_p
, "coding-system-p");
10767 DEFSYM (Qcoding_system_error
, "coding-system-error");
10768 Fput (Qcoding_system_error
, Qerror_conditions
,
10769 listn (CONSTYPE_PURE
, 2, Qcoding_system_error
, Qerror
));
10770 Fput (Qcoding_system_error
, Qerror_message
,
10771 build_pure_c_string ("Invalid coding system"));
10773 /* Intern this now in case it isn't already done.
10774 Setting this variable twice is harmless.
10775 But don't staticpro it here--that is done in alloc.c. */
10776 Qchar_table_extra_slots
= intern_c_string ("char-table-extra-slots");
10778 DEFSYM (Qtranslation_table
, "translation-table");
10779 Fput (Qtranslation_table
, Qchar_table_extra_slots
, make_number (2));
10780 DEFSYM (Qtranslation_table_id
, "translation-table-id");
10781 DEFSYM (Qtranslation_table_for_decode
, "translation-table-for-decode");
10782 DEFSYM (Qtranslation_table_for_encode
, "translation-table-for-encode");
10784 DEFSYM (Qvalid_codes
, "valid-codes");
10786 DEFSYM (Qemacs_mule
, "emacs-mule");
10788 DEFSYM (QCcategory
, ":category");
10789 DEFSYM (QCmnemonic
, ":mnemonic");
10790 DEFSYM (QCdefault_char
, ":default-char");
10791 DEFSYM (QCdecode_translation_table
, ":decode-translation-table");
10792 DEFSYM (QCencode_translation_table
, ":encode-translation-table");
10793 DEFSYM (QCpost_read_conversion
, ":post-read-conversion");
10794 DEFSYM (QCpre_write_conversion
, ":pre-write-conversion");
10795 DEFSYM (QCascii_compatible_p
, ":ascii-compatible-p");
10797 Vcoding_category_table
10798 = Fmake_vector (make_number (coding_category_max
), Qnil
);
10799 staticpro (&Vcoding_category_table
);
10800 /* Followings are target of code detection. */
10801 ASET (Vcoding_category_table
, coding_category_iso_7
,
10802 intern_c_string ("coding-category-iso-7"));
10803 ASET (Vcoding_category_table
, coding_category_iso_7_tight
,
10804 intern_c_string ("coding-category-iso-7-tight"));
10805 ASET (Vcoding_category_table
, coding_category_iso_8_1
,
10806 intern_c_string ("coding-category-iso-8-1"));
10807 ASET (Vcoding_category_table
, coding_category_iso_8_2
,
10808 intern_c_string ("coding-category-iso-8-2"));
10809 ASET (Vcoding_category_table
, coding_category_iso_7_else
,
10810 intern_c_string ("coding-category-iso-7-else"));
10811 ASET (Vcoding_category_table
, coding_category_iso_8_else
,
10812 intern_c_string ("coding-category-iso-8-else"));
10813 ASET (Vcoding_category_table
, coding_category_utf_8_auto
,
10814 intern_c_string ("coding-category-utf-8-auto"));
10815 ASET (Vcoding_category_table
, coding_category_utf_8_nosig
,
10816 intern_c_string ("coding-category-utf-8"));
10817 ASET (Vcoding_category_table
, coding_category_utf_8_sig
,
10818 intern_c_string ("coding-category-utf-8-sig"));
10819 ASET (Vcoding_category_table
, coding_category_utf_16_be
,
10820 intern_c_string ("coding-category-utf-16-be"));
10821 ASET (Vcoding_category_table
, coding_category_utf_16_auto
,
10822 intern_c_string ("coding-category-utf-16-auto"));
10823 ASET (Vcoding_category_table
, coding_category_utf_16_le
,
10824 intern_c_string ("coding-category-utf-16-le"));
10825 ASET (Vcoding_category_table
, coding_category_utf_16_be_nosig
,
10826 intern_c_string ("coding-category-utf-16-be-nosig"));
10827 ASET (Vcoding_category_table
, coding_category_utf_16_le_nosig
,
10828 intern_c_string ("coding-category-utf-16-le-nosig"));
10829 ASET (Vcoding_category_table
, coding_category_charset
,
10830 intern_c_string ("coding-category-charset"));
10831 ASET (Vcoding_category_table
, coding_category_sjis
,
10832 intern_c_string ("coding-category-sjis"));
10833 ASET (Vcoding_category_table
, coding_category_big5
,
10834 intern_c_string ("coding-category-big5"));
10835 ASET (Vcoding_category_table
, coding_category_ccl
,
10836 intern_c_string ("coding-category-ccl"));
10837 ASET (Vcoding_category_table
, coding_category_emacs_mule
,
10838 intern_c_string ("coding-category-emacs-mule"));
10839 /* Followings are NOT target of code detection. */
10840 ASET (Vcoding_category_table
, coding_category_raw_text
,
10841 intern_c_string ("coding-category-raw-text"));
10842 ASET (Vcoding_category_table
, coding_category_undecided
,
10843 intern_c_string ("coding-category-undecided"));
10845 DEFSYM (Qinsufficient_source
, "insufficient-source");
10846 DEFSYM (Qinvalid_source
, "invalid-source");
10847 DEFSYM (Qinterrupted
, "interrupted");
10848 DEFSYM (Qcoding_system_define_form
, "coding-system-define-form");
10850 defsubr (&Scoding_system_p
);
10851 defsubr (&Sread_coding_system
);
10852 defsubr (&Sread_non_nil_coding_system
);
10853 defsubr (&Scheck_coding_system
);
10854 defsubr (&Sdetect_coding_region
);
10855 defsubr (&Sdetect_coding_string
);
10856 defsubr (&Sfind_coding_systems_region_internal
);
10857 defsubr (&Sunencodable_char_position
);
10858 defsubr (&Scheck_coding_systems_region
);
10859 defsubr (&Sdecode_coding_region
);
10860 defsubr (&Sencode_coding_region
);
10861 defsubr (&Sdecode_coding_string
);
10862 defsubr (&Sencode_coding_string
);
10863 defsubr (&Sdecode_sjis_char
);
10864 defsubr (&Sencode_sjis_char
);
10865 defsubr (&Sdecode_big5_char
);
10866 defsubr (&Sencode_big5_char
);
10867 defsubr (&Sset_terminal_coding_system_internal
);
10868 defsubr (&Sset_safe_terminal_coding_system_internal
);
10869 defsubr (&Sterminal_coding_system
);
10870 defsubr (&Sset_keyboard_coding_system_internal
);
10871 defsubr (&Skeyboard_coding_system
);
10872 defsubr (&Sfind_operation_coding_system
);
10873 defsubr (&Sset_coding_system_priority
);
10874 defsubr (&Sdefine_coding_system_internal
);
10875 defsubr (&Sdefine_coding_system_alias
);
10876 defsubr (&Scoding_system_put
);
10877 defsubr (&Scoding_system_base
);
10878 defsubr (&Scoding_system_plist
);
10879 defsubr (&Scoding_system_aliases
);
10880 defsubr (&Scoding_system_eol_type
);
10881 defsubr (&Scoding_system_priority_list
);
10883 DEFVAR_LISP ("coding-system-list", Vcoding_system_list
,
10884 doc
: /* List of coding systems.
10886 Do not alter the value of this variable manually. This variable should be
10887 updated by the functions `define-coding-system' and
10888 `define-coding-system-alias'. */);
10889 Vcoding_system_list
= Qnil
;
10891 DEFVAR_LISP ("coding-system-alist", Vcoding_system_alist
,
10892 doc
: /* Alist of coding system names.
10893 Each element is one element list of coding system name.
10894 This variable is given to `completing-read' as COLLECTION argument.
10896 Do not alter the value of this variable manually. This variable should be
10897 updated by the functions `make-coding-system' and
10898 `define-coding-system-alias'. */);
10899 Vcoding_system_alist
= Qnil
;
10901 DEFVAR_LISP ("coding-category-list", Vcoding_category_list
,
10902 doc
: /* List of coding-categories (symbols) ordered by priority.
10904 On detecting a coding system, Emacs tries code detection algorithms
10905 associated with each coding-category one by one in this order. When
10906 one algorithm agrees with a byte sequence of source text, the coding
10907 system bound to the corresponding coding-category is selected.
10909 Don't modify this variable directly, but use `set-coding-system-priority'. */);
10913 Vcoding_category_list
= Qnil
;
10914 for (i
= coding_category_max
- 1; i
>= 0; i
--)
10915 Vcoding_category_list
10916 = Fcons (AREF (Vcoding_category_table
, i
),
10917 Vcoding_category_list
);
10920 DEFVAR_LISP ("coding-system-for-read", Vcoding_system_for_read
,
10921 doc
: /* Specify the coding system for read operations.
10922 It is useful to bind this variable with `let', but do not set it globally.
10923 If the value is a coding system, it is used for decoding on read operation.
10924 If not, an appropriate element is used from one of the coding system alists.
10925 There are three such tables: `file-coding-system-alist',
10926 `process-coding-system-alist', and `network-coding-system-alist'. */);
10927 Vcoding_system_for_read
= Qnil
;
10929 DEFVAR_LISP ("coding-system-for-write", Vcoding_system_for_write
,
10930 doc
: /* Specify the coding system for write operations.
10931 Programs bind this variable with `let', but you should not set it globally.
10932 If the value is a coding system, it is used for encoding of output,
10933 when writing it to a file and when sending it to a file or subprocess.
10935 If this does not specify a coding system, an appropriate element
10936 is used from one of the coding system alists.
10937 There are three such tables: `file-coding-system-alist',
10938 `process-coding-system-alist', and `network-coding-system-alist'.
10939 For output to files, if the above procedure does not specify a coding system,
10940 the value of `buffer-file-coding-system' is used. */);
10941 Vcoding_system_for_write
= Qnil
;
10943 DEFVAR_LISP ("last-coding-system-used", Vlast_coding_system_used
,
10945 Coding system used in the latest file or process I/O. */);
10946 Vlast_coding_system_used
= Qnil
;
10948 DEFVAR_LISP ("last-code-conversion-error", Vlast_code_conversion_error
,
10950 Error status of the last code conversion.
10952 When an error was detected in the last code conversion, this variable
10953 is set to one of the following symbols.
10954 `insufficient-source'
10958 `insufficient-memory'
10959 When no error was detected, the value doesn't change. So, to check
10960 the error status of a code conversion by this variable, you must
10961 explicitly set this variable to nil before performing code
10963 Vlast_code_conversion_error
= Qnil
;
10965 DEFVAR_BOOL ("inhibit-eol-conversion", inhibit_eol_conversion
,
10967 *Non-nil means always inhibit code conversion of end-of-line format.
10968 See info node `Coding Systems' and info node `Text and Binary' concerning
10969 such conversion. */);
10970 inhibit_eol_conversion
= 0;
10972 DEFVAR_BOOL ("inherit-process-coding-system", inherit_process_coding_system
,
10974 Non-nil means process buffer inherits coding system of process output.
10975 Bind it to t if the process output is to be treated as if it were a file
10976 read from some filesystem. */);
10977 inherit_process_coding_system
= 0;
10979 DEFVAR_LISP ("file-coding-system-alist", Vfile_coding_system_alist
,
10981 Alist to decide a coding system to use for a file I/O operation.
10982 The format is ((PATTERN . VAL) ...),
10983 where PATTERN is a regular expression matching a file name,
10984 VAL is a coding system, a cons of coding systems, or a function symbol.
10985 If VAL is a coding system, it is used for both decoding and encoding
10987 If VAL is a cons of coding systems, the car part is used for decoding,
10988 and the cdr part is used for encoding.
10989 If VAL is a function symbol, the function must return a coding system
10990 or a cons of coding systems which are used as above. The function is
10991 called with an argument that is a list of the arguments with which
10992 `find-operation-coding-system' was called. If the function can't decide
10993 a coding system, it can return `undecided' so that the normal
10994 code-detection is performed.
10996 See also the function `find-operation-coding-system'
10997 and the variable `auto-coding-alist'. */);
10998 Vfile_coding_system_alist
= Qnil
;
11000 DEFVAR_LISP ("process-coding-system-alist", Vprocess_coding_system_alist
,
11002 Alist to decide a coding system to use for a process I/O operation.
11003 The format is ((PATTERN . VAL) ...),
11004 where PATTERN is a regular expression matching a program name,
11005 VAL is a coding system, a cons of coding systems, or a function symbol.
11006 If VAL is a coding system, it is used for both decoding what received
11007 from the program and encoding what sent to the program.
11008 If VAL is a cons of coding systems, the car part is used for decoding,
11009 and the cdr part is used for encoding.
11010 If VAL is a function symbol, the function must return a coding system
11011 or a cons of coding systems which are used as above.
11013 See also the function `find-operation-coding-system'. */);
11014 Vprocess_coding_system_alist
= Qnil
;
11016 DEFVAR_LISP ("network-coding-system-alist", Vnetwork_coding_system_alist
,
11018 Alist to decide a coding system to use for a network I/O operation.
11019 The format is ((PATTERN . VAL) ...),
11020 where PATTERN is a regular expression matching a network service name
11021 or is a port number to connect to,
11022 VAL is a coding system, a cons of coding systems, or a function symbol.
11023 If VAL is a coding system, it is used for both decoding what received
11024 from the network stream and encoding what sent to the network stream.
11025 If VAL is a cons of coding systems, the car part is used for decoding,
11026 and the cdr part is used for encoding.
11027 If VAL is a function symbol, the function must return a coding system
11028 or a cons of coding systems which are used as above.
11030 See also the function `find-operation-coding-system'. */);
11031 Vnetwork_coding_system_alist
= Qnil
;
11033 DEFVAR_LISP ("locale-coding-system", Vlocale_coding_system
,
11034 doc
: /* Coding system to use with system messages.
11035 Also used for decoding keyboard input on X Window system. */);
11036 Vlocale_coding_system
= Qnil
;
11038 /* The eol mnemonics are reset in startup.el system-dependently. */
11039 DEFVAR_LISP ("eol-mnemonic-unix", eol_mnemonic_unix
,
11041 *String displayed in mode line for UNIX-like (LF) end-of-line format. */);
11042 eol_mnemonic_unix
= build_pure_c_string (":");
11044 DEFVAR_LISP ("eol-mnemonic-dos", eol_mnemonic_dos
,
11046 *String displayed in mode line for DOS-like (CRLF) end-of-line format. */);
11047 eol_mnemonic_dos
= build_pure_c_string ("\\");
11049 DEFVAR_LISP ("eol-mnemonic-mac", eol_mnemonic_mac
,
11051 *String displayed in mode line for MAC-like (CR) end-of-line format. */);
11052 eol_mnemonic_mac
= build_pure_c_string ("/");
11054 DEFVAR_LISP ("eol-mnemonic-undecided", eol_mnemonic_undecided
,
11056 *String displayed in mode line when end-of-line format is not yet determined. */);
11057 eol_mnemonic_undecided
= build_pure_c_string (":");
11059 DEFVAR_LISP ("enable-character-translation", Venable_character_translation
,
11061 *Non-nil enables character translation while encoding and decoding. */);
11062 Venable_character_translation
= Qt
;
11064 DEFVAR_LISP ("standard-translation-table-for-decode",
11065 Vstandard_translation_table_for_decode
,
11066 doc
: /* Table for translating characters while decoding. */);
11067 Vstandard_translation_table_for_decode
= Qnil
;
11069 DEFVAR_LISP ("standard-translation-table-for-encode",
11070 Vstandard_translation_table_for_encode
,
11071 doc
: /* Table for translating characters while encoding. */);
11072 Vstandard_translation_table_for_encode
= Qnil
;
11074 DEFVAR_LISP ("charset-revision-table", Vcharset_revision_table
,
11075 doc
: /* Alist of charsets vs revision numbers.
11076 While encoding, if a charset (car part of an element) is found,
11077 designate it with the escape sequence identifying revision (cdr part
11078 of the element). */);
11079 Vcharset_revision_table
= Qnil
;
11081 DEFVAR_LISP ("default-process-coding-system",
11082 Vdefault_process_coding_system
,
11083 doc
: /* Cons of coding systems used for process I/O by default.
11084 The car part is used for decoding a process output,
11085 the cdr part is used for encoding a text to be sent to a process. */);
11086 Vdefault_process_coding_system
= Qnil
;
11088 DEFVAR_LISP ("latin-extra-code-table", Vlatin_extra_code_table
,
11090 Table of extra Latin codes in the range 128..159 (inclusive).
11091 This is a vector of length 256.
11092 If Nth element is non-nil, the existence of code N in a file
11093 \(or output of subprocess) doesn't prevent it to be detected as
11094 a coding system of ISO 2022 variant which has a flag
11095 `accept-latin-extra-code' t (e.g. iso-latin-1) on reading a file
11096 or reading output of a subprocess.
11097 Only 128th through 159th elements have a meaning. */);
11098 Vlatin_extra_code_table
= Fmake_vector (make_number (256), Qnil
);
11100 DEFVAR_LISP ("select-safe-coding-system-function",
11101 Vselect_safe_coding_system_function
,
11103 Function to call to select safe coding system for encoding a text.
11105 If set, this function is called to force a user to select a proper
11106 coding system which can encode the text in the case that a default
11107 coding system used in each operation can't encode the text. The
11108 function should take care that the buffer is not modified while
11109 the coding system is being selected.
11111 The default value is `select-safe-coding-system' (which see). */);
11112 Vselect_safe_coding_system_function
= Qnil
;
11114 DEFVAR_BOOL ("coding-system-require-warning",
11115 coding_system_require_warning
,
11116 doc
: /* Internal use only.
11117 If non-nil, on writing a file, `select-safe-coding-system-function' is
11118 called even if `coding-system-for-write' is non-nil. The command
11119 `universal-coding-system-argument' binds this variable to t temporarily. */);
11120 coding_system_require_warning
= 0;
11123 DEFVAR_BOOL ("inhibit-iso-escape-detection",
11124 inhibit_iso_escape_detection
,
11126 If non-nil, Emacs ignores ISO-2022 escape sequences during code detection.
11128 When Emacs reads text, it tries to detect how the text is encoded.
11129 This code detection is sensitive to escape sequences. If Emacs sees
11130 a valid ISO-2022 escape sequence, it assumes the text is encoded in one
11131 of the ISO2022 encodings, and decodes text by the corresponding coding
11132 system (e.g. `iso-2022-7bit').
11134 However, there may be a case that you want to read escape sequences in
11135 a file as is. In such a case, you can set this variable to non-nil.
11136 Then the code detection will ignore any escape sequences, and no text is
11137 detected as encoded in some ISO-2022 encoding. The result is that all
11138 escape sequences become visible in a buffer.
11140 The default value is nil, and it is strongly recommended not to change
11141 it. That is because many Emacs Lisp source files that contain
11142 non-ASCII characters are encoded by the coding system `iso-2022-7bit'
11143 in Emacs's distribution, and they won't be decoded correctly on
11144 reading if you suppress escape sequence detection.
11146 The other way to read escape sequences in a file without decoding is
11147 to explicitly specify some coding system that doesn't use ISO-2022
11148 escape sequence (e.g., `latin-1') on reading by \\[universal-coding-system-argument]. */);
11149 inhibit_iso_escape_detection
= 0;
11151 DEFVAR_BOOL ("inhibit-null-byte-detection",
11152 inhibit_null_byte_detection
,
11153 doc
: /* If non-nil, Emacs ignores null bytes on code detection.
11154 By default, Emacs treats it as binary data, and does not attempt to
11155 decode it. The effect is as if you specified `no-conversion' for
11158 Set this to non-nil when a regular text happens to include null bytes.
11159 Examples are Index nodes of Info files and null-byte delimited output
11160 from GNU Find and GNU Grep. Emacs will then ignore the null bytes and
11161 decode text as usual. */);
11162 inhibit_null_byte_detection
= 0;
11164 DEFVAR_BOOL ("disable-ascii-optimization", disable_ascii_optimization
,
11165 doc
: /* If non-nil, Emacs does not optimize code decoder for ASCII files.
11166 Internal use only. Removed after the experimental optimizer gets stable. */);
11167 disable_ascii_optimization
= 0;
11169 DEFVAR_LISP ("translation-table-for-input", Vtranslation_table_for_input
,
11170 doc
: /* Char table for translating self-inserting characters.
11171 This is applied to the result of input methods, not their input.
11172 See also `keyboard-translate-table'.
11174 Use of this variable for character code unification was rendered
11175 obsolete in Emacs 23.1 and later, since Unicode is now the basis of
11176 internal character representation. */);
11177 Vtranslation_table_for_input
= Qnil
;
11180 Lisp_Object args
[coding_arg_undecided_max
];
11181 Lisp_Object plist
[16];
11184 for (i
= 0; i
< coding_arg_undecided_max
; i
++)
11187 plist
[0] = intern_c_string (":name");
11188 plist
[1] = args
[coding_arg_name
] = Qno_conversion
;
11189 plist
[2] = intern_c_string (":mnemonic");
11190 plist
[3] = args
[coding_arg_mnemonic
] = make_number ('=');
11191 plist
[4] = intern_c_string (":coding-type");
11192 plist
[5] = args
[coding_arg_coding_type
] = Qraw_text
;
11193 plist
[6] = intern_c_string (":ascii-compatible-p");
11194 plist
[7] = args
[coding_arg_ascii_compatible_p
] = Qt
;
11195 plist
[8] = intern_c_string (":default-char");
11196 plist
[9] = args
[coding_arg_default_char
] = make_number (0);
11197 plist
[10] = intern_c_string (":for-unibyte");
11198 plist
[11] = args
[coding_arg_for_unibyte
] = Qt
;
11199 plist
[12] = intern_c_string (":docstring");
11200 plist
[13] = build_pure_c_string ("Do no conversion.\n\
11202 When you visit a file with this coding, the file is read into a\n\
11203 unibyte buffer as is, thus each byte of a file is treated as a\n\
11205 plist
[14] = intern_c_string (":eol-type");
11206 plist
[15] = args
[coding_arg_eol_type
] = Qunix
;
11207 args
[coding_arg_plist
] = Flist (16, plist
);
11208 Fdefine_coding_system_internal (coding_arg_max
, args
);
11210 plist
[1] = args
[coding_arg_name
] = Qundecided
;
11211 plist
[3] = args
[coding_arg_mnemonic
] = make_number ('-');
11212 plist
[5] = args
[coding_arg_coding_type
] = Qundecided
;
11213 /* This is already set.
11214 plist[7] = args[coding_arg_ascii_compatible_p] = Qt; */
11215 plist
[8] = intern_c_string (":charset-list");
11216 plist
[9] = args
[coding_arg_charset_list
] = Fcons (Qascii
, Qnil
);
11217 plist
[11] = args
[coding_arg_for_unibyte
] = Qnil
;
11218 plist
[13] = build_pure_c_string ("No conversion on encoding, automatic conversion on decoding.");
11219 plist
[15] = args
[coding_arg_eol_type
] = Qnil
;
11220 args
[coding_arg_plist
] = Flist (16, plist
);
11221 Fdefine_coding_system_internal (coding_arg_undecided_max
, args
);
11224 setup_coding_system (Qno_conversion
, &safe_terminal_coding
);
11229 for (i
= 0; i
< coding_category_max
; i
++)
11230 Fset (AREF (Vcoding_category_table
, i
), Qno_conversion
);
11232 #if defined (DOS_NT)
11233 system_eol_type
= Qdos
;
11235 system_eol_type
= Qunix
;
11237 staticpro (&system_eol_type
);
11241 emacs_strerror (int error_number
)
11245 synchronize_system_messages_locale ();
11246 str
= strerror (error_number
);
11248 if (! NILP (Vlocale_coding_system
))
11250 Lisp_Object dec
= code_convert_string_norecord (build_string (str
),
11251 Vlocale_coding_system
,
11253 str
= SSDATA (dec
);