#include <config.h>
#include <stdio.h>
+#ifdef HAVE_WCHAR_H
+#include <wchar.h>
+#endif /* HAVE_WCHAR_H */
+
#include "lisp.h"
#include "character.h"
#include "buffer.h"
static Lisp_Object Qcoding_system, Qeol_type;
static Lisp_Object Qcoding_aliases;
Lisp_Object Qunix, Qdos;
+static Lisp_Object Qmac;
Lisp_Object Qbuffer_file_coding_system;
static Lisp_Object Qpost_read_conversion, Qpre_write_conversion;
static Lisp_Object Qdefault_char;
Lisp_Object Qstart_process, Qopen_network_stream;
static Lisp_Object Qtarget_idx;
-static Lisp_Object Qinsufficient_source, Qinconsistent_eol, Qinvalid_source;
-static Lisp_Object Qinterrupted, Qinsufficient_memory;
+static Lisp_Object Qinsufficient_source, Qinvalid_source, Qinterrupted;
/* If a symbol has this property, evaluate the value to define the
symbol as a coding system. */
case CODING_RESULT_INSUFFICIENT_SRC:
Vlast_code_conversion_error = Qinsufficient_source;
break;
- case CODING_RESULT_INCONSISTENT_EOL:
- Vlast_code_conversion_error = Qinconsistent_eol;
- break;
case CODING_RESULT_INVALID_SRC:
Vlast_code_conversion_error = Qinvalid_source;
break;
case CODING_RESULT_INTERRUPT:
Vlast_code_conversion_error = Qinterrupted;
break;
- case CODING_RESULT_INSUFFICIENT_MEM:
- Vlast_code_conversion_error = Qinsufficient_memory;
- break;
case CODING_RESULT_INSUFFICIENT_DST:
/* Don't record this error in Vlast_code_conversion_error
because it happens just temporarily and is resolved when the
GPT -= gap_head_used, GPT_BYTE -= gap_head_used;
}
else
- {
- Lisp_Object this_buffer;
-
- this_buffer = Fcurrent_buffer ();
- set_buffer_internal (XBUFFER (coding->dst_object));
- make_gap (bytes);
- set_buffer_internal (XBUFFER (this_buffer));
- }
+ make_gap_1 (XBUFFER (coding->dst_object), bytes);
}
}
if (single_shifting)
break;
- check_extra_latin:
- if (! VECTORP (Vlatin_extra_code_table)
- || NILP (AREF (Vlatin_extra_code_table, c)))
- {
- rejected = CATEGORY_MASK_ISO;
- break;
- }
- if (CODING_ISO_FLAGS (&coding_categories[coding_category_iso_8_1])
- & CODING_ISO_FLAG_LATIN_EXTRA)
- found |= CATEGORY_MASK_ISO_8_1;
- else
- rejected |= CATEGORY_MASK_ISO_8_1;
- rejected |= CATEGORY_MASK_ISO_8_2;
- break;
+ goto check_extra_latin;
default:
if (c < 0)
}
break;
}
+ check_extra_latin:
+ if (! VECTORP (Vlatin_extra_code_table)
+ || NILP (AREF (Vlatin_extra_code_table, c)))
+ {
+ rejected = CATEGORY_MASK_ISO;
+ break;
+ }
+ if (CODING_ISO_FLAGS (&coding_categories[coding_category_iso_8_1])
+ & CODING_ISO_FLAG_LATIN_EXTRA)
+ found |= CATEGORY_MASK_ISO_8_1;
+ else
+ rejected |= CATEGORY_MASK_ISO_8_1;
+ rejected |= CATEGORY_MASK_ISO_8_2;
+ break;
}
}
detect_info->rejected |= CATEGORY_MASK_ISO;
*charbuf++ = c < 0 ? -c : ASCII_BYTE_P (c) ? c : BYTE8_TO_CHAR (c);
char_offset++;
coding->errors++;
+ /* Reset the invocation and designation status to the safest
+ one; i.e. designate ASCII to the graphic register 0, and
+ invoke that register to the graphic plane 0. This typically
+ helps the case that an designation sequence for ASCII "ESC (
+ B" is somehow broken (e.g. broken by a newline). */
+ CODING_ISO_INVOCATION (coding, 0) = 0;
+ CODING_ISO_DESIGNATION (coding, 0) = charset_ascii;
+ charset_id_0 = charset_ascii;
continue;
break_loop:
#define EOL_SEEN_CR 2
#define EOL_SEEN_CRLF 4
+
+static Lisp_Object adjust_coding_eol_type (struct coding_system *coding,
+ int eol_seen);
+
+
+/* Return the number of ASCII characters at the head of the source.
+ By side effects, set coding->head_ascii and coding->eol_seen. The
+ value of coding->eol_seen is "logical or" of EOL_SEEN_LF,
+ EOL_SEEN_CR, and EOL_SEEN_CRLF, but the value is reliable only when
+ all the source bytes are ASCII. */
+
+static int
+check_ascii (struct coding_system *coding)
+{
+ const unsigned char *src, *end;
+ Lisp_Object eol_type = CODING_ID_EOL_TYPE (coding->id);
+ int eol_seen;
+
+ eol_seen = (VECTORP (eol_type) ? EOL_SEEN_NONE
+ : EQ (eol_type, Qunix) ? EOL_SEEN_LF
+ : EQ (eol_type, Qdos) ? EOL_SEEN_CRLF
+ : EOL_SEEN_CR);
+ coding_set_source (coding);
+ src = coding->source;
+ end = src + coding->src_bytes;
+
+ if (inhibit_eol_conversion
+ || eol_seen != EOL_SEEN_NONE)
+ {
+ /* We don't have to check EOL format. */
+ while (src < end && !( *src & 0x80)) src++;
+ if (inhibit_eol_conversion)
+ {
+ eol_seen = EOL_SEEN_LF;
+ adjust_coding_eol_type (coding, eol_seen);
+ }
+ }
+ else
+ {
+ end--; /* We look ahead one byte for "CR LF". */
+ while (src < end)
+ {
+ int c = *src;
+
+ if (c & 0x80)
+ break;
+ src++;
+ if (c == '\r')
+ {
+ if (*src == '\n')
+ {
+ eol_seen |= EOL_SEEN_CRLF;
+ src++;
+ }
+ else
+ eol_seen |= EOL_SEEN_CR;
+ }
+ else if (c == '\n')
+ eol_seen |= EOL_SEEN_LF;
+ }
+ if (src == end)
+ {
+ int c = *src;
+
+ /* All bytes but the last one C are ASCII. */
+ if (! (c & 0x80))
+ {
+ if (c == '\r')
+ eol_seen |= EOL_SEEN_CR;
+ else if (c == '\n')
+ eol_seen |= EOL_SEEN_LF;
+ src++;
+ }
+ }
+ }
+ coding->head_ascii = src - coding->source;
+ coding->eol_seen = eol_seen;
+ return (coding->head_ascii);
+}
+
+
+/* Return the number of characters at the source if all the bytes are
+ valid UTF-8 (of Unicode range). Otherwise, return -1. By side
+ effects, update coding->eol_seen. The value of coding->eol_seen is
+ "logical or" of EOL_SEEN_LF, EOL_SEEN_CR, and EOL_SEEN_CRLF, but
+ the value is reliable only when all the source bytes are valid
+ UTF-8. */
+
+static int
+check_utf_8 (struct coding_system *coding)
+{
+ const unsigned char *src, *end;
+ int eol_seen = coding->eol_seen;
+ int nchars = coding->head_ascii;
+
+ if (coding->head_ascii < 0)
+ check_ascii (coding);
+ else
+ coding_set_source (coding);
+ src = coding->source + coding->head_ascii;
+ /* We look ahead one byte for CR LF. */
+ end = coding->source + coding->src_bytes - 1;
+
+ while (src < end)
+ {
+ int c = *src;
+
+ if (UTF_8_1_OCTET_P (*src))
+ {
+ src++;
+ if (c < 0x20)
+ {
+ if (c == '\r')
+ {
+ if (*src == '\n')
+ {
+ eol_seen |= EOL_SEEN_CRLF;
+ src++;
+ nchars++;
+ }
+ else
+ eol_seen |= EOL_SEEN_CR;
+ }
+ else if (c == '\n')
+ eol_seen |= EOL_SEEN_LF;
+ }
+ }
+ else if (UTF_8_2_OCTET_LEADING_P (c))
+ {
+ if (c < 0xC2 /* overlong sequence */
+ || src + 1 >= end
+ || ! UTF_8_EXTRA_OCTET_P (src[1]))
+ return -1;
+ src += 2;
+ }
+ else if (UTF_8_3_OCTET_LEADING_P (c))
+ {
+ if (src + 2 >= end
+ || ! (UTF_8_EXTRA_OCTET_P (src[1])
+ && UTF_8_EXTRA_OCTET_P (src[2])))
+ return -1;
+ c = (((c & 0xF) << 12)
+ | ((src[1] & 0x3F) << 6) | (src[2] & 0x3F));
+ if (c < 0x800 /* overlong sequence */
+ || (c >= 0xd800 && c < 0xe000)) /* surrogates (invalid) */
+ return -1;
+ src += 3;
+ }
+ else if (UTF_8_4_OCTET_LEADING_P (c))
+ {
+ if (src + 3 >= end
+ || ! (UTF_8_EXTRA_OCTET_P (src[1])
+ && UTF_8_EXTRA_OCTET_P (src[2])
+ && UTF_8_EXTRA_OCTET_P (src[3])))
+ return -1;
+ c = (((c & 0x7) << 18) | ((src[1] & 0x3F) << 12)
+ | ((src[2] & 0x3F) << 6) | (src[3] & 0x3F));
+ if (c < 0x10000 /* overlong sequence */
+ || c >= 0x110000) /* non-Unicode character */
+ return -1;
+ src += 4;
+ }
+ else
+ return -1;
+ nchars++;
+ }
+
+ if (src == end)
+ {
+ if (! UTF_8_1_OCTET_P (*src))
+ return -1;
+ nchars++;
+ if (*src == '\r')
+ eol_seen |= EOL_SEEN_CR;
+ else if (*src == '\n')
+ eol_seen |= EOL_SEEN_LF;
+ }
+ coding->eol_seen = eol_seen;
+ return nchars;
+}
+
+
/* Detect how end-of-line of a text of length SRC_BYTES pointed by
SOURCE is encoded. If CATEGORY is one of
coding_category_utf_16_XXXX, assume that CR and LF are encoded by
Lisp_Object eol_type;
eol_type = CODING_ID_EOL_TYPE (coding->id);
+ if (! VECTORP (eol_type))
+ /* Already adjusted. */
+ return eol_type;
if (eol_seen & EOL_SEEN_LF)
{
coding->id = CODING_SYSTEM_ID (AREF (eol_type, 0));
coding_set_source (coding);
src_end = coding->source + coding->src_bytes;
- coding->head_ascii = 0;
/* If we have not yet decided the text encoding type, detect it
now. */
struct coding_detection_info detect_info;
bool null_byte_found = 0, eight_bit_found = 0;
+ coding->head_ascii = 0;
+ coding->eol_seen = EOL_SEEN_NONE;
detect_info.checked = detect_info.found = detect_info.rejected = 0;
for (src = coding->source; src < src_end; src++)
{
if (eight_bit_found)
break;
}
+ else if (! disable_ascii_optimization
+ && ! inhibit_eol_conversion)
+ {
+ if (c == '\r')
+ {
+ if (src < src_end && src[1] == '\n')
+ {
+ coding->eol_seen |= EOL_SEEN_CRLF;
+ src++;
+ if (! eight_bit_found)
+ coding->head_ascii++;
+ }
+ else
+ coding->eol_seen |= EOL_SEEN_CR;
+ }
+ else if (c == '\n')
+ {
+ coding->eol_seen |= EOL_SEEN_LF;
+ }
+ }
+
if (! eight_bit_found)
coding->head_ascii++;
}
coding_systems
= AREF (CODING_ID_ATTRS (coding->id), coding_attr_utf_bom);
detect_info.found = detect_info.rejected = 0;
- coding->head_ascii = 0;
- if (CONSP (coding_systems)
- && detect_coding_utf_8 (coding, &detect_info))
+ if (check_ascii (coding) == coding->src_bytes)
{
- if (detect_info.found & CATEGORY_MASK_UTF_8_SIG)
- setup_coding_system (XCAR (coding_systems), coding);
- else
- setup_coding_system (XCDR (coding_systems), coding);
+ int head_ascii = coding->head_ascii;
+
+ if (coding->eol_seen != EOL_SEEN_NONE)
+ adjust_coding_eol_type (coding, coding->eol_seen);
+ setup_coding_system (XCDR (coding_systems), coding);
+ coding->head_ascii = head_ascii;
+ }
+ else
+ {
+ if (CONSP (coding_systems)
+ && detect_coding_utf_8 (coding, &detect_info))
+ {
+ if (detect_info.found & CATEGORY_MASK_UTF_8_SIG)
+ setup_coding_system (XCAR (coding_systems), coding);
+ else
+ setup_coding_system (XCDR (coding_systems), coding);
+ }
}
}
else if (XINT (CODING_ATTR_CATEGORY (CODING_ID_ATTRS (coding->id)))
= AREF (CODING_ID_ATTRS (coding->id), coding_attr_utf_bom);
detect_info.found = detect_info.rejected = 0;
coding->head_ascii = 0;
+ coding->eol_seen = EOL_SEEN_NONE;
if (CONSP (coding_systems)
&& detect_coding_utf_16 (coding, &detect_info))
{
produced = dst - (coding->destination + coding->produced);
if (BUFFERP (coding->dst_object) && produced_chars > 0)
- insert_from_gap (produced_chars, produced);
+ insert_from_gap (produced_chars, produced, 0);
coding->produced += produced;
coding->produced_char += produced_chars;
return carryover;
#define ALLOC_CONVERSION_WORK_AREA(coding) \
do { \
- int size = CHARBUF_SIZE; \
- \
- coding->charbuf = NULL; \
- while (size > 1024) \
- { \
- coding->charbuf = alloca (sizeof (int) * size); \
- if (coding->charbuf) \
- break; \
- size >>= 1; \
- } \
- if (! coding->charbuf) \
- { \
- record_conversion_result (coding, CODING_RESULT_INSUFFICIENT_MEM); \
- return; \
- } \
- coding->charbuf_size = size; \
+ coding->charbuf = SAFE_ALLOCA (CHARBUF_SIZE * sizeof (int)); \
+ coding->charbuf_size = CHARBUF_SIZE; \
} while (0)
int carryover;
int i;
+ USE_SAFE_ALLOCA;
+
if (BUFFERP (coding->src_object)
&& coding->src_pos > 0
&& coding->src_pos < GPT
bset_undo_list (current_buffer, undo_list);
record_insert (coding->dst_pos, coding->produced_char);
}
+
+ SAFE_FREE ();
}
int max_lookup;
struct ccl_spec cclspec;
+ USE_SAFE_ALLOCA;
+
attrs = CODING_ID_ATTRS (coding->id);
if (coding->encoder == encode_coding_raw_text)
translation_table = Qnil, max_lookup = 0;
} while (coding->consumed_char < coding->src_chars);
if (BUFFERP (coding->dst_object) && coding->produced_char > 0)
- insert_from_gap (coding->produced_char, coding->produced);
+ insert_from_gap (coding->produced_char, coding->produced, 0);
+
+ SAFE_FREE ();
}
ptrdiff_t count = SPECPDL_INDEX ();
Lisp_Object attrs;
- code_conversion_save (0, 0);
-
coding->src_object = Fcurrent_buffer ();
coding->src_chars = chars;
coding->src_bytes = bytes;
if (CODING_REQUIRE_DETECTION (coding))
detect_coding (coding);
+ attrs = CODING_ID_ATTRS (coding->id);
+ if (! disable_ascii_optimization
+ && ! coding->src_multibyte
+ && ! NILP (CODING_ATTR_ASCII_COMPAT (attrs))
+ && NILP (CODING_ATTR_POST_READ (attrs))
+ && NILP (get_translation_table (attrs, 0, NULL)))
+ {
+ chars = coding->head_ascii;
+ if (chars < 0)
+ chars = check_ascii (coding);
+ if (chars != bytes)
+ {
+ if (EQ (CODING_ATTR_TYPE (attrs), Qutf_8))
+ chars = check_utf_8 (coding);
+ else
+ chars = -1;
+ }
+ if (chars >= 0)
+ {
+ if (coding->eol_seen != EOL_SEEN_NONE)
+ adjust_coding_eol_type (coding, coding->eol_seen);
+
+ if (coding->eol_seen == EOL_SEEN_CR)
+ {
+ unsigned char *src_end = GAP_END_ADDR;
+ unsigned char *src = src_end - coding->src_bytes;
+
+ while (src < src_end)
+ {
+ if (*src++ == '\r')
+ src[-1] = '\n';
+ }
+ }
+ else if (coding->eol_seen == EOL_SEEN_CRLF)
+ {
+ unsigned char *src = GAP_END_ADDR;
+ unsigned char *src_beg = src - coding->src_bytes;
+ unsigned char *dst = src;
+ ptrdiff_t diff;
+
+ while (src_beg < src)
+ {
+ *--dst = *--src;
+ if (*src == '\n' && src > src_beg && src[-1] == '\r')
+ src--;
+ }
+ diff = dst - src;
+ bytes -= diff;
+ chars -= diff;
+ }
+ coding->produced = bytes;
+ coding->produced_char = chars;
+ insert_from_gap (chars, bytes, 1);
+ return;
+ }
+ }
+ code_conversion_save (0, 0);
coding->mode |= CODING_MODE_LAST_BLOCK;
current_buffer->text->inhibit_shrinking = 1;
decode_coding (coding);
current_buffer->text->inhibit_shrinking = 0;
- attrs = CODING_ID_ATTRS (coding->id);
if (! NILP (CODING_ATTR_POST_READ (attrs)))
{
ptrdiff_t prev_Z = Z, prev_Z_BYTE = Z_BYTE;
set_buffer_internal (XBUFFER (coding->dst_object));
if (dst_bytes < coding->produced)
{
+ eassert (coding->produced > 0);
destination = xrealloc (destination, coding->produced);
- if (! destination)
- {
- record_conversion_result (coding,
- CODING_RESULT_INSUFFICIENT_MEM);
- unbind_to (count, Qnil);
- return;
- }
if (BEGV < GPT && GPT < BEGV + coding->produced_char)
move_gap_both (BEGV, BEGV_BYTE);
memcpy (destination, BEGV_ADDR, coding->produced);
return code_convert_string_norecord (str, Qutf_16le, 0);
}
+Lisp_Object
+from_unicode_buffer (const wchar_t* wstr)
+{
+ return from_unicode (
+ make_unibyte_string (
+ (char*) wstr,
+ /* we get one of the two final 0 bytes for free. */
+ 1 + sizeof (wchar_t) * wcslen (wstr)));
+}
+
wchar_t *
to_unicode (Lisp_Object str, Lisp_Object *buf)
{
*buf = code_convert_string_norecord (str, Qutf_16le, 1);
- /* We need to make a another copy (in addition to the one made by
+ /* We need to make another copy (in addition to the one made by
code_convert_string_norecord) to ensure that the final string is
_doubly_ zero terminated --- that is, that the string is
terminated by two zero bytes and one utf-16le null character.
ptrdiff_t from, to;
ptrdiff_t from_byte, to_byte;
- CHECK_NUMBER_COERCE_MARKER (start);
- CHECK_NUMBER_COERCE_MARKER (end);
-
validate_region (&start, &end);
from = XINT (start), to = XINT (end);
from_byte = CHAR_TO_BYTE (from);
ptrdiff_t from, from_byte, to, to_byte;
Lisp_Object src_object;
- CHECK_NUMBER_COERCE_MARKER (start);
- CHECK_NUMBER_COERCE_MARKER (end);
if (NILP (coding_system))
coding_system = Qno_conversion;
else
int i;
memcpy (buf, SDATA (SYMBOL_NAME (base)), base_name_len);
- subsidiaries = Fmake_vector (make_number (3), Qnil);
+ subsidiaries = make_uninit_vector (3);
for (i = 0; i < 3; i++)
{
strcpy (buf + base_name_len, suffixes[i]);
CHECK_VECTOR (initial);
for (i = 0; i < 4; i++)
{
- val = Faref (initial, make_number (i));
+ val = AREF (initial, i);
if (! NILP (val))
{
struct charset *charset;
this_name = AREF (eol_type, i);
this_aliases = Fcons (this_name, Qnil);
this_eol_type = (i == 0 ? Qunix : i == 1 ? Qdos : Qmac);
- this_spec = Fmake_vector (make_number (3), attrs);
+ this_spec = make_uninit_vector (3);
+ ASET (this_spec, 0, attrs);
ASET (this_spec, 1, this_aliases);
ASET (this_spec, 2, this_eol_type);
Fputhash (this_name, this_spec, Vcoding_system_hash_table);
}
}
- spec_vec = Fmake_vector (make_number (3), attrs);
+ spec_vec = make_uninit_vector (3);
+ ASET (spec_vec, 0, attrs);
ASET (spec_vec, 1, aliases);
ASET (spec_vec, 2, eol_type);
DEFSYM (Qeol_type, "eol-type");
DEFSYM (Qunix, "unix");
DEFSYM (Qdos, "dos");
+ DEFSYM (Qmac, "mac");
DEFSYM (Qbuffer_file_coding_system, "buffer-file-coding-system");
DEFSYM (Qpost_read_conversion, "post-read-conversion");
intern_c_string ("coding-category-undecided"));
DEFSYM (Qinsufficient_source, "insufficient-source");
- DEFSYM (Qinconsistent_eol, "inconsistent-eol");
DEFSYM (Qinvalid_source, "invalid-source");
DEFSYM (Qinterrupted, "interrupted");
- DEFSYM (Qinsufficient_memory, "insufficient-memory");
DEFSYM (Qcoding_system_define_form, "coding-system-define-form");
defsubr (&Scoding_system_p);
The other way to read escape sequences in a file without decoding is
to explicitly specify some coding system that doesn't use ISO-2022
-escape sequence (e.g `latin-1') on reading by \\[universal-coding-system-argument]. */);
+escape sequence (e.g., `latin-1') on reading by \\[universal-coding-system-argument]. */);
inhibit_iso_escape_detection = 0;
DEFVAR_BOOL ("inhibit-null-byte-detection",
decode text as usual. */);
inhibit_null_byte_detection = 0;
+ DEFVAR_BOOL ("disable-ascii-optimization", disable_ascii_optimization,
+ doc: /* If non-nil, Emacs does not optimize code decoder for ASCII files.
+Internal use only. Removed after the experimental optimizer gets stable. */);
+ disable_ascii_optimization = 0;
+
DEFVAR_LISP ("translation-table-for-input", Vtranslation_table_for_input,
doc: /* Char table for translating self-inserting characters.
This is applied to the result of input methods, not their input.