+ (setq
+ tt
+ (catch 'return
+ (when (eq modifier 'TEMPLATE_TAIL)
+ (setf (js2-token-beg token) (1- js2-ts-cursor))
+ (throw 'return (js2-get-string-or-template-token ?` token)))
+ (while t
+ ;; Eat whitespace, possibly sensitive to newlines.
+ (setq continue t)
+ (while continue
+ (setq c (js2-get-char))
+ (cond
+ ((eq c js2-EOF_CHAR)
+ (js2-unget-char)
+ (js2-ts-set-char-token-bounds token)
+ (throw 'return js2-EOF))
+ ((eq c ?\n)
+ (js2-ts-set-char-token-bounds token)
+ (setq js2-ts-dirty-line nil)
+ (throw 'return js2-EOL))
+ ((not (js2-js-space-p c))
+ (if (/= c ?-) ; in case end of HTML comment
+ (setq js2-ts-dirty-line t))
+ (setq continue nil))))
+ ;; Assume the token will be 1 char - fixed up below.
+ (js2-ts-set-char-token-bounds token)
+ (when (eq c ?@)
+ (throw 'return js2-XMLATTR))
+ ;; identifier/keyword/instanceof?
+ ;; watch out for starting with a <backslash>
+ (cond
+ ((eq c ?\\)
+ (setq c (js2-get-char))
+ (if (eq c ?u)
+ (setq identifier-start t
+ is-unicode-escape-start t
+ js2-ts-string-buffer nil)
+ (setq identifier-start nil)
+ (js2-unget-char)
+ (setq c ?\\)))
+ (t
+ (when (setq identifier-start (js2-identifier-start-p c))
+ (setq js2-ts-string-buffer nil)
+ (js2-add-to-string c))))
+ (when identifier-start
+ (setq contains-escape is-unicode-escape-start)
+ (catch 'break
+ (while t
+ (if is-unicode-escape-start
+ ;; strictly speaking we should probably push-back
+ ;; all the bad characters if the <backslash>uXXXX
+ ;; sequence is malformed. But since there isn't a
+ ;; correct context(is there?) for a bad Unicode
+ ;; escape sequence in an identifier, we can report
+ ;; an error here.
+ (progn
+ (setq escape-val 0)
+ (dotimes (_ 4)
+ (setq c (js2-get-char)
+ escape-val (js2-x-digit-to-int c escape-val))
+ ;; Next check takes care of c < 0 and bad escape
+ (if (cl-minusp escape-val)
+ (throw 'break nil)))
+ (if (cl-minusp escape-val)
+ (js2-report-scan-error "msg.invalid.escape" t))
+ (js2-add-to-string escape-val)
+ (setq is-unicode-escape-start nil))
+ (setq c (js2-get-char))
+ (cond
+ ((eq c ?\\)
+ (setq c (js2-get-char))
+ (if (eq c ?u)
+ (setq is-unicode-escape-start t
+ contains-escape t)
+ (js2-report-scan-error "msg.illegal.character" t)))
+ (t
+ (if (or (eq c js2-EOF_CHAR)
+ (not (js2-identifier-part-p c)))
+ (throw 'break nil))
+ (js2-add-to-string c))))))
+ (js2-unget-char)
+ (setf str (js2-collect-string js2-ts-string-buffer)
+ (js2-token-end token) js2-ts-cursor)
+ ;; FIXME: Invalid in ES5 and ES6, see
+ ;; https://bugzilla.mozilla.org/show_bug.cgi?id=694360
+ ;; Probably should just drop this conditional.
+ (unless contains-escape
+ ;; OPT we shouldn't have to make a string (object!) to
+ ;; check if it's a keyword.
+ ;; Return the corresponding token if it's a keyword
+ (when (and (not (eq modifier 'KEYWORD_IS_NAME))
+ (setq result (js2-string-to-keyword str)))
+ (if (and (< js2-language-version 170)
+ (memq result '(js2-LET js2-YIELD)))
+ ;; LET and YIELD are tokens only in 1.7 and later
+ (setq result 'js2-NAME))
+ (when (eq result 'js2-RESERVED)
+ (setf (js2-token-string token) str))
+ (throw 'return (js2-tt-code result))))
+ ;; If we want to intern these as Rhino does, just use (intern str)
+ (setf (js2-token-string token) str)
+ (throw 'return js2-NAME)) ; end identifier/kwd check
+ ;; is it a number?
+ (when (or (js2-digit-p c)
+ (and (eq c ?.) (js2-digit-p (js2-peek-char))))
+ (setq js2-ts-string-buffer nil
+ base 10)
+ (when (eq c ?0)
+ (setq c (js2-get-char))
+ (cond
+ ((or (eq c ?x) (eq c ?X))
+ (setq base 16)
+ (setq c (js2-get-char)))
+ ((and (or (eq c ?b) (eq c ?B))
+ (>= js2-language-version 200))
+ (setq base 2)
+ (setq c (js2-get-char)))
+ ((and (or (eq c ?o) (eq c ?O))
+ (>= js2-language-version 200))
+ (setq base 8)
+ (setq c (js2-get-char)))
+ ((js2-digit-p c)
+ (setq base 'maybe-8))
+ (t
+ (js2-add-to-string ?0))))
+ (cond
+ ((eq base 16)
+ (if (> 0 (js2-x-digit-to-int c 0))
+ (js2-report-scan-error "msg.missing.hex.digits")
+ (while (<= 0 (js2-x-digit-to-int c 0))
+ (js2-add-to-string c)
+ (setq c (js2-get-char)))))
+ ((eq base 2)
+ (if (not (memq c '(?0 ?1)))
+ (js2-report-scan-error "msg.missing.binary.digits")
+ (while (memq c '(?0 ?1))
+ (js2-add-to-string c)
+ (setq c (js2-get-char)))))
+ ((eq base 8)
+ (if (or (> ?0 c) (< ?7 c))
+ (js2-report-scan-error "msg.missing.octal.digits")
+ (while (and (<= ?0 c) (>= ?7 c))
+ (js2-add-to-string c)
+ (setq c (js2-get-char)))))
+ (t
+ (while (and (<= ?0 c) (<= c ?9))
+ ;; We permit 08 and 09 as decimal numbers, which
+ ;; makes our behavior a superset of the ECMA
+ ;; numeric grammar. We might not always be so
+ ;; permissive, so we warn about it.
+ (when (and (eq base 'maybe-8) (>= c ?8))
+ (js2-report-warning "msg.bad.octal.literal"
+ (if (eq c ?8) "8" "9"))
+ (setq base 10))
+ (js2-add-to-string c)
+ (setq c (js2-get-char)))
+ (when (eq base 'maybe-8)
+ (setq base 8))))
+ (when (and (eq base 10) (memq c '(?. ?e ?E)))
+ (when (eq c ?.)
+ (cl-loop do
+ (js2-add-to-string c)
+ (setq c (js2-get-char))
+ while (js2-digit-p c)))
+ (when (memq c '(?e ?E))
+ (js2-add-to-string c)
+ (setq c (js2-get-char))
+ (when (memq c '(?+ ?-))
+ (js2-add-to-string c)
+ (setq c (js2-get-char)))
+ (unless (js2-digit-p c)
+ (js2-report-scan-error "msg.missing.exponent" t))
+ (cl-loop do
+ (js2-add-to-string c)
+ (setq c (js2-get-char))
+ while (js2-digit-p c))))
+ (js2-unget-char)
+ (let ((str (js2-set-string-from-buffer token)))
+ (setf (js2-token-number token)
+ (js2-string-to-number str base)))
+ (throw 'return js2-NUMBER))
+ ;; is it a string?
+ (when (or (memq c '(?\" ?\'))
+ (and (>= js2-language-version 200)
+ (= c ?`)))
+ (throw 'return
+ (js2-get-string-or-template-token c token)))
+ (js2-ts-return token
+ (cl-case c
+ (?\;
+ (throw 'return js2-SEMI))
+ (?\[
+ (throw 'return js2-LB))
+ (?\]
+ (throw 'return js2-RB))
+ (?{
+ (throw 'return js2-LC))
+ (?}
+ (throw 'return js2-RC))
+ (?\(
+ (throw 'return js2-LP))
+ (?\)
+ (throw 'return js2-RP))
+ (?,
+ (throw 'return js2-COMMA))
+ (??
+ (throw 'return js2-HOOK))
+ (?:
+ (if (js2-match-char ?:)
+ js2-COLONCOLON
+ (throw 'return js2-COLON)))
+ (?.
+ (if (js2-match-char ?.)
+ (if (js2-match-char ?.)
+ js2-TRIPLEDOT js2-DOTDOT)
+ (if (js2-match-char ?\()
+ js2-DOTQUERY
+ (throw 'return js2-DOT))))
+ (?|
+ (if (js2-match-char ?|)
+ (throw 'return js2-OR)
+ (if (js2-match-char ?=)
+ js2-ASSIGN_BITOR
+ (throw 'return js2-BITOR))))
+ (?^
+ (if (js2-match-char ?=)
+ js2-ASSIGN_BITOR
+ (throw 'return js2-BITXOR)))
+ (?&
+ (if (js2-match-char ?&)
+ (throw 'return js2-AND)
+ (if (js2-match-char ?=)
+ js2-ASSIGN_BITAND
+ (throw 'return js2-BITAND))))
+ (?=
+ (if (js2-match-char ?=)
+ (if (js2-match-char ?=)
+ js2-SHEQ
+ (throw 'return js2-EQ))
+ (if (js2-match-char ?>)
+ (js2-ts-return token js2-ARROW)
+ (throw 'return js2-ASSIGN))))
+ (?!
+ (if (js2-match-char ?=)
+ (if (js2-match-char ?=)
+ js2-SHNE
+ js2-NE)
+ (throw 'return js2-NOT)))
+ (?<
+ ;; NB:treat HTML begin-comment as comment-till-eol
+ (when (js2-match-char ?!)
+ (when (js2-match-char ?-)
+ (when (js2-match-char ?-)
+ (js2-skip-line)
+ (setf (js2-token-comment-type (js2-current-token)) 'html)
+ (throw 'return js2-COMMENT)))
+ (js2-unget-char))
+ (if (js2-match-char ?<)
+ (if (js2-match-char ?=)
+ js2-ASSIGN_LSH
+ js2-LSH)
+ (if (js2-match-char ?=)
+ js2-LE
+ (throw 'return js2-LT))))
+ (?>
+ (if (js2-match-char ?>)
+ (if (js2-match-char ?>)
+ (if (js2-match-char ?=)
+ js2-ASSIGN_URSH
+ js2-URSH)
+ (if (js2-match-char ?=)
+ js2-ASSIGN_RSH
+ js2-RSH))
+ (if (js2-match-char ?=)
+ js2-GE
+ (throw 'return js2-GT))))
+ (?*
+ (if (js2-match-char ?=)
+ js2-ASSIGN_MUL
+ (throw 'return js2-MUL)))
+ (?/
+ ;; is it a // comment?
+ (when (js2-match-char ?/)
+ (setf (js2-token-beg token) (- js2-ts-cursor 2))
+ (js2-skip-line)
+ (setf (js2-token-comment-type token) 'line)
+ ;; include newline so highlighting goes to end of window
+ (cl-incf (js2-token-end token))
+ (throw 'return js2-COMMENT))
+ ;; is it a /* comment?
+ (when (js2-match-char ?*)
+ (setf look-for-slash nil
+ (js2-token-beg token) (- js2-ts-cursor 2)
+ (js2-token-comment-type token)
+ (if (js2-match-char ?*)
+ (progn
+ (setq look-for-slash t)
+ 'jsdoc)
+ 'block))
+ (while t
+ (setq c (js2-get-char))
+ (cond
+ ((eq c js2-EOF_CHAR)
+ (setf (js2-token-end token) (1- js2-ts-cursor))
+ (js2-report-error "msg.unterminated.comment")
+ (throw 'return js2-COMMENT))
+ ((eq c ?*)
+ (setq look-for-slash t))
+ ((eq c ?/)
+ (if look-for-slash
+ (js2-ts-return token js2-COMMENT)))
+ (t
+ (setf look-for-slash nil
+ (js2-token-end token) js2-ts-cursor)))))
+ (if (js2-match-char ?=)
+ js2-ASSIGN_DIV
+ (throw 'return js2-DIV)))
+ (?#
+ (when js2-skip-preprocessor-directives
+ (js2-skip-line)
+ (setf (js2-token-comment-type token) 'preprocessor
+ (js2-token-end token) js2-ts-cursor)
+ (throw 'return js2-COMMENT))
+ (throw 'return js2-ERROR))
+ (?%
+ (if (js2-match-char ?=)
+ js2-ASSIGN_MOD
+ (throw 'return js2-MOD)))
+ (?~
+ (throw 'return js2-BITNOT))
+ (?+
+ (if (js2-match-char ?=)
+ js2-ASSIGN_ADD
+ (if (js2-match-char ?+)
+ js2-INC
+ (throw 'return js2-ADD))))
+ (?-
+ (cond
+ ((js2-match-char ?=)
+ (setq c js2-ASSIGN_SUB))
+ ((js2-match-char ?-)
+ (unless js2-ts-dirty-line
+ ;; treat HTML end-comment after possible whitespace
+ ;; after line start as comment-until-eol
+ (when (js2-match-char ?>)
+ (js2-skip-line)
+ (setf (js2-token-comment-type (js2-current-token)) 'html)
+ (throw 'return js2-COMMENT)))
+ (setq c js2-DEC))
+ (t
+ (setq c js2-SUB)))
+ (setq js2-ts-dirty-line t)
+ c)
+ (otherwise
+ (js2-report-scan-error "msg.illegal.character")))))))
+ (setf (js2-token-type token) tt)
+ token))
+
+(defun js2-get-string-or-template-token (quote-char token)
+ ;; We attempt to accumulate a string the fast way, by
+ ;; building it directly out of the reader. But if there
+ ;; are any escaped characters in the string, we revert to
+ ;; building it out of a string buffer.
+ (let ((c (js2-get-char))
+ js2-ts-string-buffer
+ nc c1 val escape-val)
+ (catch 'break
+ (while (/= c quote-char)
+ (catch 'continue
+ (when (eq c js2-EOF_CHAR)