;;; semantic/lex.el --- Lexical Analyzer builder
-;; Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007,
-;; 2008, 2009, 2010 Free Software Foundation, Inc.
+;; Copyright (C) 1999-2015 Free Software Foundation, Inc.
;; Author: Eric M. Ludlam <zappo@gnu.org>
;;
;; %type <punctuation> syntax "\\(\\s.\\|\\s$\\|\\s'\\)+" matchdatatype string
;;
-;; ;; Some punctuations based on the type defines above
+;; ;; Some punctuation based on the type defines above
;;
;; %token <punctuation> NOT "!"
;; %token <punctuation> NOTEQ "!="
(if default
(message
"*Warning* default value of <%s> tokens changed to %S, was %S"
- type default token))
+ type token default))
(setq default token)))
;; Ensure the default matching spec is the first one.
(semantic-lex-type-set type (cons default (nreverse alist))))
(symbol)
(whitespace)
)
- "An alist of of semantic token types.
+ "An alist of semantic token types.
As of December 2001 (semantic 1.4beta13), this variable is not used in
any code. The only use is to refer to the doc-string from elsewhere.
(semantic-overlay-put o 'face 'highlight)
o))
-(defsubst semantic-lex-debug-break (token)
- "Break during lexical analysis at TOKEN."
- (when semantic-lex-debug
- (let ((o nil))
- (unwind-protect
- (progn
- (when token
- (setq o (semantic-lex-highlight-token token)))
- (semantic-read-event
- (format "%S :: SPC - continue" token))
- )
- (when o
- (semantic-overlay-delete o))))))
-
;;; Lexical analyzer creation
;;
;; Code for creating a lex function from lists of analyzers.
start position of the block, and STREAM is the list of tokens in that
block.")
-(defvar semantic-lex-reset-hooks nil
+(define-obsolete-variable-alias 'semantic-lex-reset-hooks
+ 'semantic-lex-reset-functions "24.3")
+(defvar semantic-lex-reset-functions nil
"Abnormal hook used by major-modes to reset lexical analyzers.
Hook functions are called with START and END values for the
current lexical pass. Should be set with `add-hook', specifying
;;(defvar semantic-lex-timeout 5
;; "*Number of sections of lexing before giving up.")
+(defsubst semantic-lex-debug-break (token)
+ "Break during lexical analysis at TOKEN."
+ (when semantic-lex-debug
+ (let ((o nil))
+ (unwind-protect
+ (progn
+ (when token
+ (setq o (semantic-lex-highlight-token token)))
+ (semantic-read-event
+ (format "%S :: Depth: %d :: SPC - continue" token semantic-lex-current-depth))
+ )
+ (when o
+ (semantic-overlay-delete o))))))
+
(defmacro define-lex (name doc &rest analyzers)
"Create a new lexical analyzer with NAME.
DOC is a documentation string describing this analyzer.
;; Make sure the state of block parsing starts over.
(setq semantic-lex-block-streams nil)
;; Allow specialty reset items.
- (run-hook-with-args 'semantic-lex-reset-hooks start end)
+ (run-hook-with-args 'semantic-lex-reset-functions start end)
;; Lexing state.
(let* (;(starttime (current-time))
(starting-position (point))
;; Return the token stream
(nreverse semantic-lex-token-stream))))
\f
-;;; Collapsed block tokens delimited by any tokens.
-;;
-(defun semantic-lex-start-block (syntax)
- "Mark the last read token as the beginning of a SYNTAX block."
- (if (or (not semantic-lex-maximum-depth)
- (< semantic-lex-current-depth semantic-lex-maximum-depth))
- (setq semantic-lex-current-depth (1+ semantic-lex-current-depth))
- (push (list syntax (car semantic-lex-token-stream))
- semantic-lex-block-stack)))
-
-(defun semantic-lex-end-block (syntax)
- "Process the end of a previously marked SYNTAX block.
-That is, collapse the tokens inside that block, including the
-beginning and end of block tokens, into a high level block token of
-class SYNTAX.
-The token at beginning of block is the one marked by a previous call
-to `semantic-lex-start-block'. The current token is the end of block.
-The collapsed tokens are saved in `semantic-lex-block-streams'."
- (if (null semantic-lex-block-stack)
- (setq semantic-lex-current-depth (1- semantic-lex-current-depth))
- (let* ((stream semantic-lex-token-stream)
- (blk (pop semantic-lex-block-stack))
- (bstream (cdr blk))
- (first (car bstream))
- (last (pop stream)) ;; The current token mark the EOBLK
- tok)
- (if (not (eq (car blk) syntax))
- ;; SYNTAX doesn't match the syntax of the current block in
- ;; the stack. So we encountered the end of the SYNTAX block
- ;; before the end of the current one in the stack which is
- ;; signaled unterminated.
- (semantic-lex-unterminated-syntax-detected (car blk))
- ;; Move tokens found inside the block from the main stream
- ;; into a separate block stream.
- (while (and stream (not (eq (setq tok (pop stream)) first)))
- (push tok bstream))
- ;; The token marked as beginning of block was not encountered.
- ;; This should not happen!
- (or (eq tok first)
- (error "Token %S not found at beginning of block `%s'"
- first syntax))
- ;; Save the block stream for future reuse, to avoid to redo
- ;; the lexical analysis of the block content!
- ;; Anchor the block stream with its start position, so we can
- ;; use: (cdr (assq start semantic-lex-block-streams)) to
- ;; quickly retrieve the lexical stream associated to a block.
- (setcar blk (semantic-lex-token-start first))
- (setcdr blk (nreverse bstream))
- (push blk semantic-lex-block-streams)
- ;; In the main stream, replace the tokens inside the block by
- ;; a high level block token of class SYNTAX.
- (setq semantic-lex-token-stream stream)
- (semantic-lex-push-token
- (semantic-lex-token
- syntax (car blk) (semantic-lex-token-end last)))
- ))))
-\f
;;; Lexical token API
;;
;; Functions for accessing parts of a token. Use these functions
(semantic-lex-token-end semlist)
depth))
\f
+;;; Collapsed block tokens delimited by any tokens.
+;;
+(defun semantic-lex-start-block (syntax)
+ "Mark the last read token as the beginning of a SYNTAX block."
+ (if (or (not semantic-lex-maximum-depth)
+ (< semantic-lex-current-depth semantic-lex-maximum-depth))
+ (setq semantic-lex-current-depth (1+ semantic-lex-current-depth))
+ (push (list syntax (car semantic-lex-token-stream))
+ semantic-lex-block-stack)))
+
+(defun semantic-lex-end-block (syntax)
+ "Process the end of a previously marked SYNTAX block.
+That is, collapse the tokens inside that block, including the
+beginning and end of block tokens, into a high level block token of
+class SYNTAX.
+The token at beginning of block is the one marked by a previous call
+to `semantic-lex-start-block'. The current token is the end of block.
+The collapsed tokens are saved in `semantic-lex-block-streams'."
+ (if (null semantic-lex-block-stack)
+ (setq semantic-lex-current-depth (1- semantic-lex-current-depth))
+ (let* ((stream semantic-lex-token-stream)
+ (blk (pop semantic-lex-block-stack))
+ (bstream (cdr blk))
+ (first (car bstream))
+ (last (pop stream)) ;; The current token mark the EOBLK
+ tok)
+ (if (not (eq (car blk) syntax))
+ ;; SYNTAX doesn't match the syntax of the current block in
+ ;; the stack. So we encountered the end of the SYNTAX block
+ ;; before the end of the current one in the stack which is
+ ;; signaled unterminated.
+ (semantic-lex-unterminated-syntax-detected (car blk))
+ ;; Move tokens found inside the block from the main stream
+ ;; into a separate block stream.
+ (while (and stream (not (eq (setq tok (pop stream)) first)))
+ (push tok bstream))
+ ;; The token marked as beginning of block was not encountered.
+ ;; This should not happen!
+ (or (eq tok first)
+ (error "Token %S not found at beginning of block `%s'"
+ first syntax))
+ ;; Save the block stream for future reuse, to avoid to redo
+ ;; the lexical analysis of the block content!
+ ;; Anchor the block stream with its start position, so we can
+ ;; use: (cdr (assq start semantic-lex-block-streams)) to
+ ;; quickly retrieve the lexical stream associated to a block.
+ (setcar blk (semantic-lex-token-start first))
+ (setcdr blk (nreverse bstream))
+ (push blk semantic-lex-block-streams)
+ ;; In the main stream, replace the tokens inside the block by
+ ;; a high level block token of class SYNTAX.
+ (setq semantic-lex-token-stream stream)
+ (semantic-lex-push-token
+ (semantic-lex-token
+ syntax (car blk) (semantic-lex-token-end last)))
+ ))))
+\f
;;; Analyzer creation macros
;;
;; An individual analyzer is a condition and code that goes with it.
))
))
((setq match (assoc text ',clist))
- (setq semantic-lex-current-depth (1- semantic-lex-current-depth))
- (semantic-lex-push-token
- (semantic-lex-token
- (nth 1 match)
- (match-beginning 0) (match-end 0)))))))
+ (if (> semantic-lex-current-depth 0)
+ (progn
+ (setq semantic-lex-current-depth (1- semantic-lex-current-depth))
+ (semantic-lex-push-token
+ (semantic-lex-token
+ (nth 1 match)
+ (match-beginning 0) (match-end 0)))))))))
)))
\f
;;; Analyzers
(if (eq (semantic-lex-token-class (car semantic-lex-token-stream))
'whitespace)
;; Merge whitespace tokens together if they are adjacent. Two
- ;; whitespace tokens may be sperated by a comment which is not in
+ ;; whitespace tokens may be separated by a comment which is not in
;; the token stream.
(setcdr (semantic-lex-token-bounds (car semantic-lex-token-stream))
(match-end 0))
(if (eq (semantic-lex-token-class (car semantic-lex-token-stream))
'whitespace)
;; Merge whitespace tokens together if they are adjacent. Two
- ;; whitespace tokens may be sperated by a comment which is not in
+ ;; whitespace tokens may be separated by a comment which is not in
;; the token stream.
(progn
(setq semantic-lex-end-point (match-end 0))
(define-lex-analyzer semantic-lex-punctuation-type
"Detect and create a punctuation type token.
-Recognized punctuations are defined in the current table of lexical
+Recognized punctuation is defined in the current table of lexical
types, as the value of the `punctuation' token type."
(and (looking-at "\\(\\s.\\|\\s$\\|\\s'\\)+")
(let* ((key (match-string 0))
))
(define-lex-simple-regex-analyzer semantic-lex-open-paren
- "Detect and create an open parenthisis token."
+ "Detect and create an open parenthesis token."
"\\s(" 'open-paren 0 (setq semantic-lex-current-depth (1+ semantic-lex-current-depth)))
(define-lex-simple-regex-analyzer semantic-lex-close-paren
- "Detect and create a close paren token."
+ "Detect and create a close parenthesis token."
"\\s)" 'close-paren 0 (setq semantic-lex-current-depth (1- semantic-lex-current-depth)))
(define-lex-regex-analyzer semantic-lex-string
;;; Comment lexer
;;
;; Predefined lexers that could be used instead of creating new
-;; analyers.
+;; analyzers.
(define-lex semantic-comment-lexer
"A simple lexical analyzer that handles comments.
;;; Lexical Safety
;;
;; The semantic lexers, unlike other lexers, can throw errors on
-;; unbalanced syntax. Since editing is all about changeging test
+;; unbalanced syntax. Since editing is all about changing text
;; we need to provide a convenient way to protect against syntactic
;; inequalities.
nil))))
;; Great Sadness. Assume that FORMS execute within the
;; confines of the current buffer only! Mark this thing
- ;; unparseable iff the special symbol was thrown. This
+ ;; unparsable iff the special symbol was thrown. This
;; will prevent future calls from parsing, but will allow
;; then to still return the cache.
(when ,ret
(semantic-alias-obsolete 'semantic-flex-map-keywords 'semantic-lex-map-keywords "23.2")
(semantic-alias-obsolete 'semantic-flex-keywords 'semantic-lex-keywords "23.2")
(semantic-alias-obsolete 'semantic-flex-buffer 'semantic-lex-buffer "23.2")
-(semantic-alias-obsolete 'semantic-flex-list 'semantic-lex-list "23.2")
+(semantic-alias-obsolete 'semantic-flex-list 'semantic-lex-list "23.2")
;; This simple scanner uses the syntax table to generate a stream of
;; simple tokens of the form:
;; objects boundary.
(defvar semantic-flex-tokens semantic-lex-tokens
- "An alist of of semantic token types.
+ "An alist of semantic token types.
See variable `semantic-lex-tokens'.")
(defvar semantic-flex-unterminated-syntax-end-function