character is prohibited by CODING_ISO_FLAG_SAFE. */
#define CODING_INHIBIT_CHARACTER_SUBSTITUTION '?'
+/* UTF-8 section */
+#define CODING_UTF_8_BOM(coding) \
+ ((coding)->spec.utf_8_bom)
/* UTF-16 section */
#define CODING_UTF_16_BOM(coding) \
coding_category_iso_8_2,
coding_category_iso_7_else,
coding_category_iso_8_else,
- coding_category_utf_8,
+ coding_category_utf_8_auto,
+ coding_category_utf_8_nosig,
+ coding_category_utf_8_sig,
coding_category_utf_16_auto,
coding_category_utf_16_be,
coding_category_utf_16_le,
#define CATEGORY_MASK_ISO_8_2 (1 << coding_category_iso_8_2)
#define CATEGORY_MASK_ISO_7_ELSE (1 << coding_category_iso_7_else)
#define CATEGORY_MASK_ISO_8_ELSE (1 << coding_category_iso_8_else)
-#define CATEGORY_MASK_UTF_8 (1 << coding_category_utf_8)
+#define CATEGORY_MASK_UTF_8_AUTO (1 << coding_category_utf_8_auto)
+#define CATEGORY_MASK_UTF_8_NOSIG (1 << coding_category_utf_8_nosig)
+#define CATEGORY_MASK_UTF_8_SIG (1 << coding_category_utf_8_sig)
#define CATEGORY_MASK_UTF_16_AUTO (1 << coding_category_utf_16_auto)
#define CATEGORY_MASK_UTF_16_BE (1 << coding_category_utf_16_be)
#define CATEGORY_MASK_UTF_16_LE (1 << coding_category_utf_16_le)
| CATEGORY_MASK_ISO_8_2 \
| CATEGORY_MASK_ISO_7_ELSE \
| CATEGORY_MASK_ISO_8_ELSE \
- | CATEGORY_MASK_UTF_8 \
+ | CATEGORY_MASK_UTF_8_AUTO \
+ | CATEGORY_MASK_UTF_8_NOSIG \
+ | CATEGORY_MASK_UTF_8_SIG \
| CATEGORY_MASK_UTF_16_AUTO \
| CATEGORY_MASK_UTF_16_BE \
| CATEGORY_MASK_UTF_16_LE \
| CATEGORY_MASK_UTF_16_BE_NOSIG \
| CATEGORY_MASK_UTF_16_LE_NOSIG)
+#define CATEGORY_MASK_UTF_8 \
+ (CATEGORY_MASK_UTF_8_AUTO \
+ | CATEGORY_MASK_UTF_8_NOSIG \
+ | CATEGORY_MASK_UTF_8_SIG)
/* List of symbols `coding-category-xxx' ordered by priority. This
variable is exposed to Emacs Lisp. */
#define UTF_8_4_OCTET_LEADING_P(c) (((c) & 0xF8) == 0xF0)
#define UTF_8_5_OCTET_LEADING_P(c) (((c) & 0xFC) == 0xF8)
+#define UTF_BOM 0xFEFF
+#define UTF_8_BOM_1 0xEF
+#define UTF_8_BOM_2 0xBB
+#define UTF_8_BOM_3 0xBF
+
static int
detect_coding_utf_8 (coding, detect_info)
struct coding_system *coding;
const unsigned char *src_end = coding->source + coding->src_bytes;
int multibytep = coding->src_multibyte;
int consumed_chars = 0;
+ int bom_found = 0;
int found = 0;
detect_info->checked |= CATEGORY_MASK_UTF_8;
break;
if (UTF_8_2_OCTET_LEADING_P (c))
{
- found = CATEGORY_MASK_UTF_8;
+ found = 1;
continue;
}
ONE_MORE_BYTE (c2);
break;
if (UTF_8_3_OCTET_LEADING_P (c))
{
- found = CATEGORY_MASK_UTF_8;
+ found = 1;
+ if (src_base == coding->source
+ && c == UTF_8_BOM_1 && c1 == UTF_8_BOM_2 && c2 == UTF_8_BOM_3)
+ bom_found = 1;
continue;
}
ONE_MORE_BYTE (c3);
break;
if (UTF_8_4_OCTET_LEADING_P (c))
{
- found = CATEGORY_MASK_UTF_8;
+ found = 1;
continue;
}
ONE_MORE_BYTE (c4);
break;
if (UTF_8_5_OCTET_LEADING_P (c))
{
- found = CATEGORY_MASK_UTF_8;
+ found = 1;
continue;
}
break;
detect_info->rejected |= CATEGORY_MASK_UTF_8;
return 0;
}
- detect_info->found |= found;
+ if (bom_found)
+ {
+ /* The first character 0xFFFE doesn't necessarily mean a BOM. */
+ detect_info->found |= CATEGORY_MASK_UTF_8_SIG | CATEGORY_MASK_UTF_8_NOSIG;
+ }
+ else
+ {
+ detect_info->rejected |= CATEGORY_MASK_UTF_8_SIG;
+ if (found)
+ detect_info->found |= CATEGORY_MASK_UTF_8_NOSIG;
+ }
return 1;
}
int *charbuf_end = coding->charbuf + coding->charbuf_size;
int consumed_chars = 0, consumed_chars_base;
int multibytep = coding->src_multibyte;
+ enum utf_bom_type bom = CODING_UTF_8_BOM (coding);
Lisp_Object attr, charset_list;
int eol_crlf = EQ (CODING_ID_EOL_TYPE (coding->id), Qdos);
int byte_after_cr = -1;
CODING_GET_INFO (coding, attr, charset_list);
+ if (bom != utf_without_bom)
+ {
+ int c1, c2, c3;
+
+ src_base = src;
+ ONE_MORE_BYTE (c1);
+ if (! UTF_8_3_OCTET_LEADING_P (c1))
+ src = src_base;
+ else
+ {
+ ONE_MORE_BYTE (c2);
+ if (! UTF_8_EXTRA_OCTET_P (c2))
+ src = src_base;
+ else
+ {
+ ONE_MORE_BYTE (c3);
+ if (! UTF_8_EXTRA_OCTET_P (c3))
+ src = src_base;
+ else
+ {
+ if ((c1 != UTF_8_BOM_1)
+ || (c2 != UTF_8_BOM_2) || (c3 != UTF_8_BOM_3))
+ src = src_base;
+ else
+ CODING_UTF_8_BOM (coding) = utf_without_bom;
+ }
+ }
+ }
+ }
+ CODING_UTF_8_BOM (coding) = utf_without_bom;
+
+
+
while (1)
{
int c, c1, c2, c3, c4, c5;
int produced_chars = 0;
int c;
+ if (CODING_UTF_8_BOM (coding) == utf_with_bom)
+ {
+ ASSURE_DESTINATION (3);
+ EMIT_THREE_BYTES (UTF_8_BOM_1, UTF_8_BOM_2, UTF_8_BOM_3);
+ CODING_UTF_8_BOM (coding) = utf_without_bom;
+ }
+
if (multibytep)
{
int safe_room = MAX_MULTIBYTE_LENGTH * 2;
int *charbuf_end = coding->charbuf + coding->charbuf_size;
int consumed_chars = 0, consumed_chars_base;
int multibytep = coding->src_multibyte;
- enum utf_16_bom_type bom = CODING_UTF_16_BOM (coding);
+ enum utf_bom_type bom = CODING_UTF_16_BOM (coding);
enum utf_16_endian_type endian = CODING_UTF_16_ENDIAN (coding);
int surrogate = CODING_UTF_16_SURROGATE (coding);
Lisp_Object attr, charset_list;
CODING_GET_INFO (coding, attr, charset_list);
- if (bom == utf_16_with_bom)
+ if (bom == utf_with_bom)
{
int c, c1, c2;
src = src_base;
coding->errors++;
}
- CODING_UTF_16_BOM (coding) = utf_16_without_bom;
+ CODING_UTF_16_BOM (coding) = utf_without_bom;
}
- else if (bom == utf_16_detect_bom)
+ else if (bom == utf_detect_bom)
{
/* We have already tried to detect BOM and failed in
detect_coding. */
- CODING_UTF_16_BOM (coding) = utf_16_without_bom;
+ CODING_UTF_16_BOM (coding) = utf_without_bom;
}
while (1)
unsigned char *dst = coding->destination + coding->produced;
unsigned char *dst_end = coding->destination + coding->dst_bytes;
int safe_room = 8;
- enum utf_16_bom_type bom = CODING_UTF_16_BOM (coding);
+ enum utf_bom_type bom = CODING_UTF_16_BOM (coding);
int big_endian = CODING_UTF_16_ENDIAN (coding) == utf_16_big_endian;
int produced_chars = 0;
Lisp_Object attrs, charset_list;
CODING_GET_INFO (coding, attrs, charset_list);
- if (bom != utf_16_without_bom)
+ if (bom != utf_without_bom)
{
ASSURE_DESTINATION (safe_room);
if (big_endian)
EMIT_TWO_BYTES (0xFE, 0xFF);
else
EMIT_TWO_BYTES (0xFF, 0xFE);
- CODING_UTF_16_BOM (coding) = utf_16_without_bom;
+ CODING_UTF_16_BOM (coding) = utf_without_bom;
}
while (charbuf < charbuf_end)
}
else if (EQ (coding_type, Qutf_8))
{
+ val = AREF (attrs, coding_attr_utf_bom);
+ CODING_UTF_8_BOM (coding) = (CONSP (val) ? utf_detect_bom
+ : EQ (val, Qt) ? utf_with_bom
+ : utf_without_bom);
coding->detector = detect_coding_utf_8;
coding->decoder = decode_coding_utf_8;
coding->encoder = encode_coding_utf_8;
coding->common_flags
|= (CODING_REQUIRE_DECODING_MASK | CODING_REQUIRE_ENCODING_MASK);
+ if (CODING_UTF_8_BOM (coding) == utf_detect_bom)
+ coding->common_flags |= CODING_REQUIRE_DETECTION_MASK;
}
else if (EQ (coding_type, Qutf_16))
{
- val = AREF (attrs, coding_attr_utf_16_bom);
- CODING_UTF_16_BOM (coding) = (CONSP (val) ? utf_16_detect_bom
- : EQ (val, Qt) ? utf_16_with_bom
- : utf_16_without_bom);
+ val = AREF (attrs, coding_attr_utf_bom);
+ CODING_UTF_16_BOM (coding) = (CONSP (val) ? utf_detect_bom
+ : EQ (val, Qt) ? utf_with_bom
+ : utf_without_bom);
val = AREF (attrs, coding_attr_utf_16_endian);
CODING_UTF_16_ENDIAN (coding) = (EQ (val, Qbig) ? utf_16_big_endian
: utf_16_little_endian);
coding->encoder = encode_coding_utf_16;
coding->common_flags
|= (CODING_REQUIRE_DECODING_MASK | CODING_REQUIRE_ENCODING_MASK);
- if (CODING_UTF_16_BOM (coding) == utf_16_detect_bom)
+ if (CODING_UTF_16_BOM (coding) == utf_detect_bom)
coding->common_flags |= CODING_REQUIRE_DETECTION_MASK;
}
else if (EQ (coding_type, Qccl))
coding_set_source (coding);
src_end = coding->source + coding->src_bytes;
+ coding->head_ascii = 0;
/* If we have not yet decided the text encoding type, detect it
now. */
int null_byte_found = 0, eight_bit_found = 0;
detect_info.checked = detect_info.found = detect_info.rejected = 0;
- coding->head_ascii = -1;
for (src = coding->source; src < src_end; src++)
{
c = *src;
if (c & 0x80)
{
eight_bit_found = 1;
- if (coding->head_ascii < 0)
- coding->head_ascii = src - coding->source;
if (null_byte_found)
break;
}
&& ! inhibit_iso_escape_detection
&& ! detect_info.checked)
{
- if (coding->head_ascii < 0)
- coding->head_ascii = src - coding->source;
if (detect_coding_iso_2022 (coding, &detect_info))
{
/* We have scanned the whole data. */
if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
- /* We didn't find an 8-bit code. We may have
- found a null-byte, but it's very rare that
- a binary file confirm to ISO-2022. */
- src = src_end;
+ {
+ /* We didn't find an 8-bit code. We may
+ have found a null-byte, but it's very
+ rare that a binary file confirm to
+ ISO-2022. */
+ src = src_end;
+ coding->head_ascii = src - coding->source;
+ }
+ detect_info.rejected |= ~CATEGORY_MASK_ISO_ESCAPE;
break;
}
}
if (eight_bit_found)
break;
}
+ if (! eight_bit_found)
+ coding->head_ascii++;
}
+ else if (! eight_bit_found)
+ coding->head_ascii++;
}
- if (coding->head_ascii < 0)
- coding->head_ascii = src - coding->source;
if (null_byte_found || eight_bit_found
|| coding->head_ascii < coding->src_bytes
break;
}
}
-
- if (i < coding_category_raw_text)
- setup_coding_system (CODING_ID_NAME (this->id), coding);
- else if (null_byte_found)
- setup_coding_system (Qno_conversion, coding);
- else if ((detect_info.rejected & CATEGORY_MASK_ANY)
- == CATEGORY_MASK_ANY)
- setup_coding_system (Qraw_text, coding);
- else if (detect_info.rejected)
- for (i = 0; i < coding_category_raw_text; i++)
- if (! (detect_info.rejected & (1 << coding_priorities[i])))
- {
- this = coding_categories + coding_priorities[i];
- setup_coding_system (CODING_ID_NAME (this->id), coding);
- break;
- }
}
+
+ if (i < coding_category_raw_text)
+ setup_coding_system (CODING_ID_NAME (this->id), coding);
+ else if (null_byte_found)
+ setup_coding_system (Qno_conversion, coding);
+ else if ((detect_info.rejected & CATEGORY_MASK_ANY)
+ == CATEGORY_MASK_ANY)
+ setup_coding_system (Qraw_text, coding);
+ else if (detect_info.rejected)
+ for (i = 0; i < coding_category_raw_text; i++)
+ if (! (detect_info.rejected & (1 << coding_priorities[i])))
+ {
+ this = coding_categories + coding_priorities[i];
+ setup_coding_system (CODING_ID_NAME (this->id), coding);
+ break;
+ }
+ }
+ }
+ else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding->id)))
+ == coding_category_utf_8_auto)
+ {
+ Lisp_Object coding_systems;
+ struct coding_detection_info detect_info;
+
+ coding_systems
+ = AREF (CODING_ID_ATTRS (coding->id), coding_attr_utf_bom);
+ detect_info.found = detect_info.rejected = 0;
+ coding->head_ascii = 0;
+ if (CONSP (coding_systems)
+ && detect_coding_utf_8 (coding, &detect_info))
+ {
+ if (detect_info.found & CATEGORY_MASK_UTF_8_SIG)
+ setup_coding_system (XCAR (coding_systems), coding);
+ else
+ setup_coding_system (XCDR (coding_systems), coding);
}
}
else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding->id)))
struct coding_detection_info detect_info;
coding_systems
- = AREF (CODING_ID_ATTRS (coding->id), coding_attr_utf_16_bom);
+ = AREF (CODING_ID_ATTRS (coding->id), coding_attr_utf_bom);
detect_info.found = detect_info.rejected = 0;
+ coding->head_ascii = 0;
if (CONSP (coding_systems)
&& detect_coding_utf_16 (coding, &detect_info))
{
}
else
{
- name = Vcode_conversion_workbuf_name;
- workbuf = Fget_buffer_create (name);
if (NILP (Vcode_conversion_reused_workbuf))
- Vcode_conversion_reused_workbuf = workbuf;
+ Vcode_conversion_reused_workbuf
+ = Fget_buffer_create (Vcode_conversion_workbuf_name);
+ workbuf = Vcode_conversion_reused_workbuf;
}
current = current_buffer;
set_buffer_internal (XBUFFER (workbuf));
coding.src_multibyte = multibytep;
coding.consumed = 0;
coding.mode |= CODING_MODE_LAST_BLOCK;
+ coding.head_ascii = 0;
detect_info.checked = detect_info.found = detect_info.rejected = 0;
struct coding_system *this;
int c, i;
- coding.head_ascii = -1;
/* Skip all ASCII bytes except for a few ISO2022 controls. */
for (; src < src_end; src++)
{
if (c & 0x80)
{
eight_bit_found = 1;
- if (coding.head_ascii < 0)
- coding.head_ascii = src - coding.source;
if (null_byte_found)
break;
}
- if (c < 0x20)
+ else if (c < 0x20)
{
if ((c == ISO_CODE_ESC || c == ISO_CODE_SI || c == ISO_CODE_SO)
&& ! inhibit_iso_escape_detection
&& ! detect_info.checked)
{
- if (coding.head_ascii < 0)
- coding.head_ascii = src - coding.source;
if (detect_coding_iso_2022 (&coding, &detect_info))
{
/* We have scanned the whole data. */
if (! (detect_info.rejected & CATEGORY_MASK_ISO_7_ELSE))
- /* We didn't find an 8-bit code. We may have
- found a null-byte, but it's very rare that
- a binary file confirm to ISO-2022. */
- src = src_end;
+ {
+ /* We didn't find an 8-bit code. We may
+ have found a null-byte, but it's very
+ rare that a binary file confirm to
+ ISO-2022. */
+ src = src_end;
+ coding.head_ascii = src - coding.source;
+ }
+ detect_info.rejected |= ~CATEGORY_MASK_ISO_ESCAPE;
break;
}
}
if (eight_bit_found)
break;
}
+ if (! eight_bit_found)
+ coding.head_ascii++;
}
+ else if (! eight_bit_found)
+ coding.head_ascii++;
}
- if (coding.head_ascii < 0)
- coding.head_ascii = src - coding.source;
if (null_byte_found || eight_bit_found
|| coding.head_ascii < coding.src_bytes
detect_info.found |= found;
}
}
+ else if (base_category == coding_category_utf_8_auto)
+ {
+ if (detect_coding_utf_8 (&coding, &detect_info))
+ {
+ struct coding_system *this;
+
+ if (detect_info.found & CATEGORY_MASK_UTF_8_SIG)
+ this = coding_categories + coding_category_utf_8_sig;
+ else
+ this = coding_categories + coding_category_utf_8_nosig;
+ val = Fcons (make_number (this->id), Qnil);
+ }
+ }
else if (base_category == coding_category_utf_16_auto)
{
if (detect_coding_utf_16 (&coding, &detect_info))
Optional 4th arguments DESTINATION specifies where the decoded text goes.
If nil, the region between START and END is replaced by the decoded text.
If buffer, the decoded text is inserted in the buffer.
-In those cases, the length of the decoded text is returned..
+In those cases, the length of the decoded text is returned.
If DESTINATION is t, the decoded text is returned.
This function sets `last-coding-system-used' to the precise coding system
Optional 4th arguments DESTINATION specifies where the encoded text goes.
If nil, the region between START and END is replace by the encoded text.
If buffer, the encoded text is inserted in the buffer.
-In those cases, the length of the encoded text is returned..
+In those cases, the length of the encoded text is returned.
If DESTINATION is t, the encoded text is returned.
This function sets `last-coding-system-used' to the precise coding system
Optional fourth arg BUFFER non-nil means that the encoded text is
inserted in BUFFER instead of returned as a string. In this case,
-the return value is the length of the encoded.text.
+the return value is the length of the encoded text.
This function sets `last-coding-system-used' to the precise coding system
used (which may be different from CODING-SYSTEM if CODING-SYSTEM is
val = XCDR (bom);
CHECK_CODING_SYSTEM (val);
}
- ASET (attrs, coding_attr_utf_16_bom, bom);
+ ASET (attrs, coding_attr_utf_bom, bom);
endian = args[coding_arg_utf16_endian];
CHECK_SYMBOL (endian);
}
else if (EQ (coding_type, Qutf_8))
{
- category = coding_category_utf_8;
+ Lisp_Object bom;
+
CODING_ATTR_ASCII_COMPAT (attrs) = Qt;
+
+ if (nargs < coding_arg_utf8_max)
+ goto short_args;
+
+ bom = args[coding_arg_utf8_bom];
+ if (! NILP (bom) && ! EQ (bom, Qt))
+ {
+ CHECK_CONS (bom);
+ val = XCAR (bom);
+ CHECK_CODING_SYSTEM (val);
+ val = XCDR (bom);
+ CHECK_CODING_SYSTEM (val);
+ }
+ ASET (attrs, coding_attr_utf_bom, bom);
+
+ category = (CONSP (bom) ? coding_category_utf_8_auto
+ : NILP (bom) ? coding_category_utf_8_nosig
+ : coding_category_utf_8_sig);
}
else if (EQ (coding_type, Qundecided))
category = coding_category_undecided;
intern ("coding-category-iso-7-else"));
ASET (Vcoding_category_table, coding_category_iso_8_else,
intern ("coding-category-iso-8-else"));
- ASET (Vcoding_category_table, coding_category_utf_8,
+ ASET (Vcoding_category_table, coding_category_utf_8_auto,
+ intern ("coding-category-utf-8-auto"));
+ ASET (Vcoding_category_table, coding_category_utf_8_nosig,
intern ("coding-category-utf-8"));
+ ASET (Vcoding_category_table, coding_category_utf_8_sig,
+ intern ("coding-category-utf-8-sig"));
ASET (Vcoding_category_table, coding_category_utf_16_be,
intern ("coding-category-utf-16-be"));
ASET (Vcoding_category_table, coding_category_utf_16_auto,
DEFVAR_LISP ("coding-system-alist", &Vcoding_system_alist,
doc: /* Alist of coding system names.
Each element is one element list of coding system name.
-This variable is given to `completing-read' as TABLE argument.
+This variable is given to `completing-read' as COLLECTION argument.
Do not alter the value of this variable manually. This variable should be
updated by the functions `make-coding-system' and
doc: /* Specify the coding system for read operations.
It is useful to bind this variable with `let', but do not set it globally.
If the value is a coding system, it is used for decoding on read operation.
-If not, an appropriate element is used from one of the coding system alists:
-There are three such tables, `file-coding-system-alist',
+If not, an appropriate element is used from one of the coding system alists.
+There are three such tables: `file-coding-system-alist',
`process-coding-system-alist', and `network-coding-system-alist'. */);
Vcoding_system_for_read = Qnil;
when writing it to a file and when sending it to a file or subprocess.
If this does not specify a coding system, an appropriate element
-is used from one of the coding system alists:
-There are three such tables, `file-coding-system-alist',
+is used from one of the coding system alists.
+There are three such tables: `file-coding-system-alist',
`process-coding-system-alist', and `network-coding-system-alist'.
For output to files, if the above procedure does not specify a coding system,
the value of `buffer-file-coding-system' is used. */);
a coding system of ISO 2022 variant which has a flag
`accept-latin-extra-code' t (e.g. iso-latin-1) on reading a file
or reading output of a subprocess.
-Only 128th through 159th elements has a meaning. */);
+Only 128th through 159th elements have a meaning. */);
Vlatin_extra_code_table = Fmake_vector (make_number (256), Qnil);
DEFVAR_LISP ("select-safe-coding-system-function",
DEFVAR_LISP ("translation-table-for-input", &Vtranslation_table_for_input,
doc: /* Char table for translating self-inserting characters.
-This is applied to the result of input methods, not their input. See also
-`keyboard-translate-table'. */);
+This is applied to the result of input methods, not their input.
+See also `keyboard-translate-table'. */);
Vtranslation_table_for_input = Qnil;
{