1 ;;; wisi.el --- Utilities for implementing an indentation/navigation engine using a generalized LALR parser
3 ;; Copyright (C) 2012, 2013 Free Software Foundation, Inc.
5 ;; Author: Stephen Leake <stephen_leake@member.fsf.org>
7 ;; URL: http://stephe-leake.org/emacs/ada-mode/emacs-ada-mode.html
8 ;; Package-Requires: ((cl-lib "0"))
10 ;; This file is part of GNU Emacs.
12 ;; GNU Emacs is free software: you can redistribute it and/or modify
13 ;; it under the terms of the GNU General Public License as published by
14 ;; the Free Software Foundation, either version 3 of the License, or
15 ;; (at your option) any later version.
17 ;; GNU Emacs is distributed in the hope that it will be useful,
18 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ;; GNU General Public License for more details.
22 ;; You should have received a copy of the GNU General Public License
23 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
28 ;;;; History: first experimental version Oct 2012
30 ;;;; indentation algorithm overview
32 ;; This design is inspired in part by experience writing a SMIE
33 ;; indentation engine for Ada, and the wisent parser.
35 ;; The general approach to indenting a given token is to find the
36 ;; start of the statement it is part of, or some other relevant point
37 ;; in the statement, and indent relative to that. So we need a parser
38 ;; that lets us find statement indent points from arbitrary places in
41 ;; The grammar for Ada as represented by the EBNF in LRM Annex P is
42 ;; not LALR(1), so we use a generalized LALR(1) parser (see
43 ;; wisi-parse, wisi-compile).
45 ;; The parser actions cache indentation and other information as text
46 ;; properties of tokens in statements.
48 ;; An indentation engine moves text in the buffer, as does user
49 ;; editing, so we can't rely on character positions remaining
50 ;; constant. So the parser actions use markers to store
51 ;; positions. Text properties also move with the text.
53 ;; The stored information includes a marker at each statement indent
54 ;; point. Thus, the indentation algorithm is: find the previous token
55 ;; with cached information, and either indent from it, or fetch from
56 ;; it the marker for a previous statement indent point, and indent
59 ;; Since we have a cache (the text properties), we need to consider
60 ;; when to invalidate it. Ideally, we invalidate only when a change to
61 ;; the buffer would change the result of a parse that crosses that
62 ;; change, or starts after that change. Changes in whitespace
63 ;; (indentation and newlines) do not affect an Ada parse. Other
64 ;; languages are sensitive to newlines (Bash for example) or
65 ;; indentation (Python). Adding comments does not change a parse,
66 ;; unless code is commented out. For now we invalidate the cache after
67 ;; the edit point if the change involves anything other than
70 ;;;; comparison to the SMIE parser
72 ;; The central problem to be solved in building the SMIE parser is
73 ;; grammar precedence conflicts; the general solution is refining
74 ;; keywords so that each new keyword can be assigned a unique
75 ;; precedence. This means ad hoc code must be written to determine the
76 ;; correct refinement for each language keyword from the surrounding
77 ;; tokens. In effect, for a complex language like Ada, the knowledge
78 ;; of the language grammar is mostly embedded in the refinement code;
79 ;; only a small amount is in the refined grammar. Implementing a SMIE
80 ;; parser for a new language involves the same amount of work as the
83 ;; Using a generalized LALR parser avoids that particular problem;
84 ;; assuming the language is already defined by a grammar, it is only a
85 ;; matter of a format change to teach the wisi parser the
86 ;; language. The problem in a wisi indentation engine is caching the
87 ;; output of the parser in a useful way, since we can't start the
88 ;; parser from arbitrary places in the code (as we can with the SMIE
89 ;; parser). A second problem is determining when to invalidate the
90 ;; cache. But these problems are independent of the language being
91 ;; parsed, so once we have one wisi indentation engine working,
92 ;; adapting it to new languages should be quite simple.
94 ;; The SMIE parser does not find the start of each statement, only the
95 ;; first language keyword in each statement; additional code must be
96 ;; written to find the statement start and indent points. The wisi
97 ;; parser finds the statement start and indent points directly.
99 ;; In SMIE, it is best if each grammar rule is a complete statement,
100 ;; so forward-sexp will traverse the entire statement. If nested
101 ;; non-terminals are used, forward-sexp may stop inside one of the
102 ;; nested non-terminals. This problem does not occur with the wisi
105 ;; A downside of the wisi parser is conflicts in the grammar; they can
106 ;; be much more difficult to resolve than in the SMIE parser. The
107 ;; generalized parser helps by handling conflicts, but it does so by
108 ;; running multiple parsers in parallel, persuing each choice in the
109 ;; conflict. If the conflict is due to a genuine ambiguity, both paths
110 ;; will succeed, which causes the parse to fail, since it is not clear
111 ;; which set of text properties to store. Even if one branch
112 ;; ultimately fails, running parallel parsers over large sections of
113 ;; code is slow. Finally, this approach can lead to exponential growth
114 ;; in the number of parsers. So grammar conflicts must still be
115 ;; analyzed and minimized.
117 ;; In addition, the complete grammar must be specified; in smie, it is
118 ;; often possible to specify a subset of the grammar.
120 ;;;; grammar compiler and parser
122 ;; Since we are using a generalized LALR(1) parser, we cannot use any
123 ;; of the wisent grammar functions. We use the OpenToken Ada package
124 ;; to compile BNF to Elisp source (similar to
125 ;; semantic-grammar-create-package), and wisi-compile-grammar to
126 ;; compile that to the parser table.
128 ;; Semantic provides a complex lexer, more complicated than we need
129 ;; for indentation. So we use the elisp lexer, which consists of
130 ;; `forward-comment', `skip-syntax-forward', and `scan-sexp'. We wrap
131 ;; that in functions that return tokens in the form wisi-parse
136 ;; 'wisi' was originally short for "wisent indentation engine", but
137 ;; now is just a name.
139 ;; not using lexical-binding because we support Emacs 23
145 (require 'wisi-parse)
146 (eval-when-compile (require 'cl-lib))
150 (defvar-local wisi-class-list nil)
151 (defvar-local wisi-keyword-table nil)
152 (defvar-local wisi-punctuation-table nil)
153 (defvar-local wisi-punctuation-table-max-length 0)
154 (defvar-local wisi-string-double-term nil) ;; string delimited by double quotes
155 (defvar-local wisi-string-quote-escape-doubled nil)
156 (defvar-local wisi-string-single-term nil) ;; string delimited by single quotes
157 (defvar-local wisi-symbol-term nil)
159 (defun wisi-forward-token (&optional text-only)
160 "Move point forward across one token, skipping leading whitespace and comments.
161 Return the corresponding token, in a format determined by TEXT-ONLY:
163 TEXT-ONLY nil: (token text start . end)
165 `token' is a token symbol (not string) from `wisi-punctuation-table',
166 `wisi-keyword-table', `wisi-string-double-term', `wisi-string-double-term' or `wisi-symbol-term'.
168 `text' is the token text from the buffer
170 `start, end' are the character positions in the buffer of the start
171 and end of the token text.
173 If at end of buffer, returns `wisent-eoi-term'."
174 (forward-comment (point-max))
175 ;; skips leading whitespace, comment, trailing whitespace.
177 (let ((start (point))
178 ;; (info "(elisp)Syntax Table Internals" "*info elisp syntax*")
179 (syntax (syntax-class (syntax-after (point))))
184 (setq token-id wisent-eoi-term))
187 ;; punctuation. Find the longest matching string in wisi-punctuation-table
189 (let ((next-point (point))
190 temp-text temp-id done)
192 (setq temp-text (buffer-substring-no-properties start (point)))
193 (setq temp-id (car (rassoc temp-text wisi-punctuation-table)))
195 (setq token-text temp-text
200 (= (- (point) start) wisi-punctuation-table-max-length))
204 (goto-char next-point)))
206 ((memq syntax '(4 5)) ;; open, close parenthesis
208 (setq token-text (buffer-substring-no-properties start (point)))
209 (setq token-id (symbol-value (intern-soft token-text wisi-keyword-table))))
212 ;; string quote, either single or double. we assume point is before the start quote, not the end quote
213 (let ((delim (char-after (point)))
214 (forward-sexp-function nil))
216 ;; point is now after the end quote; check for a doubled quote
217 (while (and wisi-string-quote-escape-doubled
218 (eq (char-after (point)) delim))
220 (setq token-text (buffer-substring-no-properties start (point)))
221 (setq token-id (if (= delim ?\") wisi-string-double-term wisi-string-single-term))))
223 (t ;; assuming word syntax
224 (skip-syntax-forward "w_'")
225 (setq token-text (buffer-substring-no-properties start (point)))
227 (or (symbol-value (intern-soft (downcase token-text) wisi-keyword-table))
232 (error (wisi-error-msg "unrecognized token '%s'" (buffer-substring-no-properties start (point)))))
236 (cons token-id (cons token-text (cons start (point)))))
239 (defun wisi-backward-token ()
240 "Move point backward across one token, skipping whitespace and comments.
241 Return (nil text start . end) - same structure as
242 wisi-forward-token, but does not look up symbol."
243 (forward-comment (- (point)))
244 ;; skips leading whitespace, comment, trailing whitespace.
246 ;; (info "(elisp)Syntax Table Internals" "*info elisp syntax*")
248 (syntax (syntax-class (syntax-after (1- (point))))))
252 ((memq syntax '(4 5)) ;; open, close parenthesis
256 ;; a string quote. we assume we are after the end quote, not the start quote
257 (let ((forward-sexp-function nil))
261 (if (zerop (skip-syntax-backward "."))
262 (skip-syntax-backward "w_'")))
264 (cons nil (cons (buffer-substring-no-properties (point) end) (cons (point) end)))
267 ;;;; token info cache
269 ;; the cache stores the results of parsing as text properties on
270 ;; keywords, for use by the indention and motion engines.
274 (:constructor wisi-cache-create)
276 nonterm;; nonterminal from parse (set by wisi-statement-action)
279 ;; terminal symbol from wisi-keyword-table or
280 ;; wisi-punctuation-table, or lower-level nonterminal from parse
281 ;; (set by wisi-statement-action)
283 last ;; pos of last char in token, relative to first (0 indexed)
286 ;; arbitrary lisp symbol, used for indentation and navigation.
287 ;; some classes are defined by wisi:
289 ;; 'block-middle - a block keyword (ie: if then else end), not at the start of a statement
291 ;; 'block-start - a block keyword at the start of a statement
293 ;; 'statement-start - the start of a statement
297 ;; others are language-specific
300 ;; Marker at the containing keyword for this token.
301 ;; A containing keyword is an indent point; the start of a
302 ;; statement, or 'begin', 'then' or 'else' for a block of
304 ;; nil only for first token in buffer
306 prev ;; marker at previous motion token in statement; nil if none
307 next ;; marker at next motion token in statement; nil if none
308 end ;; marker at token at end of current statement
311 (defvar-local wisi-cache-max 0
312 "Maximimum position in buffer where wisi token cache is valid.")
314 (defvar-local wisi-parse-table nil)
316 (defvar-local wisi-parse-failed nil
317 "Non-nil when a recent parse has failed - cleared when parse succeeds.")
319 (defvar-local wisi-parse-try nil
320 "Non-nil when parse is needed - cleared when parse succeeds.")
322 (defvar-local wisi-change-need-invalidate nil)
323 (defvar-local wisi-change-jit-lock-mode nil)
325 (defun wisi-invalidate-cache()
326 "Invalidate the wisi token cache for the current buffer.
327 Also invalidate the Emacs syntax cache."
329 (setq wisi-cache-max 0)
330 (setq wisi-parse-try t)
331 (syntax-ppss-flush-cache (point-min))
332 (with-silent-modifications
333 (remove-text-properties (point-min) (point-max) '(wisi-cache))))
335 (defun wisi-before-change (begin end)
336 "For `before-change-functions'."
337 ;; begin . end is range of text being deleted
339 ;; If jit-lock-after-change is before wisi-after-change in
340 ;; after-change-functions, it might use any invalid caches in the
343 ;; So we check for that here, and ensure it is after
344 ;; wisi-after-change, which deletes the invalid caches
345 (when (boundp 'jit-lock-mode)
346 (when (memq 'wisi-after-change (memq 'jit-lock-after-change after-change-functions))
347 (setq after-change-functions (delete 'wisi-after-change after-change-functions))
348 (add-hook 'after-change-functions 'wisi-after-change nil t)
349 (setq wisi-change-jit-lock-mode (1+ wisi-change-jit-lock-mode)))
353 ;; don't invalidate parse for whitespace, string, or comment changes
354 (let (;; (info "(elisp)Parser State")
355 (state (syntax-ppss begin)))
356 ;; syntax-ppss has moved point to "begin".
359 (nth 3 state); in string
360 (nth 4 state)); in comment
361 ;; FIXME: check that entire range is in comment or string
362 (setq wisi-change-need-invalidate nil))
365 (skip-syntax-forward " " end);; does not skip newline
367 (setq wisi-change-need-invalidate nil))
369 (t (setq wisi-change-need-invalidate t))
372 (defun wisi-after-change (begin end length)
373 "For `after-change-functions'."
374 ;; begin . end is range of text being inserted (may be empty)
375 ;; (syntax-ppss-flush-cache begin) is in before-change-functions
377 (syntax-ppss-flush-cache begin) ;; IMPROVEME: could check for whitespace
381 ;; The parse was failing, probably due to bad syntax; this change
382 ;; may have fixed it, so try reparse.
383 (setq wisi-parse-try t)
385 ;; remove 'wisi-cache on inserted text, which could have caches
386 ;; from before the failed parse, and are in any case invalid.
387 (with-silent-modifications
388 (remove-text-properties begin end '(wisi-cache)))
391 ((>= wisi-cache-max begin)
392 ;; The parse had succeeded paste the start of the inserted
395 (let ((need-invalidate t)
396 ;; (info "(elisp)Parser State")
397 (state (syntax-ppss begin)))
398 ;; syntax-ppss has moved point to "begin".
400 (wisi-change-need-invalidate
401 ;; wisi-before change determined the removed text alters the
406 (nth 3 state); in string
407 (nth 4 state)); in comment
408 ;; FIXME: insert newline in comment to create non-comment!?
409 ;; or paste a chunk of code
410 ;; => check that all of change region is comment or string
411 (setq need-invalidate nil))
414 (skip-syntax-forward " " end);; does not skip newlines
416 (setq need-invalidate nil))
422 ;; The inserted or deleted text could alter the parse
423 (wisi-invalidate-cache)
425 ;; else move cache-max by the net change length. We don't
426 ;; need to delete 'wisi-cache in the inserted text, because
427 ;; if there were any it would not pass the above.
429 (+ wisi-cache-max (- end begin length))))
434 ;; parse never attempted, or only done to before BEGIN. Just
435 ;; remove 'wisi-cache
436 (with-silent-modifications
437 (remove-text-properties begin end '(wisi-cache)))
441 (defun wisi-get-cache (pos)
442 "Return `wisi-cache' struct from the `wisi-cache' text property at POS.
443 If accessing cache at a marker for a token as set by `wisi-cache-tokens', POS must be (1- mark)."
444 (get-text-property pos 'wisi-cache))
448 0 : normal - ignore parse errors, for indenting new code
449 1 : report parse errors (for running tests)
450 2 : show parse states, position point at parse errors, debug-on-error works in parser
451 3 : also show top 10 items of parser stack.")
453 (defvar-local wisi-parse-error-msg nil)
455 (defun wisi-goto-error ()
456 "Move point to position in last error message (if any)."
457 (when (string-match ":\\([0-9]+\\):\\([0-9]+\\):" wisi-parse-error-msg)
458 (let ((line (string-to-number (match-string 1 wisi-parse-error-msg)))
459 (col (string-to-number (match-string 2 wisi-parse-error-msg))))
460 (goto-char (point-min))
461 (forward-line (1- line))
462 (forward-char col))))
464 (defun wisi-show-parse-error ()
465 "Show last wisi-parse error."
467 (if wisi-parse-failed
469 (message wisi-parse-error-msg)
471 (message "parse succeeded")))
473 (defun wisi-validate-cache (pos)
474 "Ensure cached data is valid at least up to POS in current buffer."
475 (when (and wisi-parse-try
476 (< wisi-cache-max pos))
477 (when (> wisi-debug 0)
478 (message "wisi: parsing ..."))
480 (setq wisi-parse-try nil)
481 (setq wisi-parse-error-msg nil)
483 (goto-char wisi-cache-max)
485 ;; let debugger stop in wisi-parse
487 (wisi-parse wisi-parse-table 'wisi-forward-token)
488 (setq wisi-cache-max (point))
489 (setq wisi-parse-failed nil))
490 ;; else capture errors from bad syntax, so higher level functions can try to continue
493 (wisi-parse wisi-parse-table 'wisi-forward-token)
494 (setq wisi-cache-max (point))
495 (setq wisi-parse-failed nil))
497 (setq wisi-parse-failed t)
498 (setq wisi-parse-error-msg (cdr err)))
500 (if wisi-parse-error-msg
502 (when (> wisi-debug 0)
503 (message "wisi: parsing ... error")
505 (error wisi-parse-error-msg))
507 (when (> wisi-debug 0)
508 (message "wisi: parsing ... done")))
511 (defun wisi-get-containing-cache (cache)
512 "Return cache from (wisi-cache-containing CACHE)."
513 (let ((containing (wisi-cache-containing cache)))
515 (wisi-get-cache (1- containing)))))
517 (defun wisi-cache-text (cache)
518 "Return property-less buffer substring designated by cache.
519 Point must be at cache."
520 (buffer-substring-no-properties (point) (+ (point) (wisi-cache-last cache))))
524 (defun wisi-set-end (tokens end-mark)
525 "Set END-MARK on all unset caches in TOKENS."
526 (let ((tokens-t tokens))
528 (let* ((token (pop tokens-t))
529 (region (cddr token))
532 (goto-char (car region))
533 (setq cache (wisi-get-cache (car region)))
535 (< (point) (cdr region)))
536 (if (not (wisi-cache-end cache))
537 (setf (wisi-cache-end cache) end-mark)
538 (goto-char (wisi-cache-end cache))
540 (setq cache (wisi-forward-cache))
545 (defvar wisi-tokens nil);; keep byte-compiler happy; `wisi-tokens' is bound in action created by wisi-semantic-action
546 (defun wisi-statement-action (&rest pairs)
547 "Cache information in text properties of tokens.
548 Intended as a grammar non-terminal action.
550 PAIRS is of the form [TOKEN-NUMBER CLASS] ... where TOKEN-NUMBER
551 is the (1 indexed) token number in the production, CLASS is the wisi class of
552 that token. Use in a grammar action as:
553 (wisi-statement-action 1 'statement-start 7 'statement-end)"
557 (override-start nil))
559 (let* ((number (1- (pop pairs)))
560 (region (cddr (nth number wisi-tokens)));; wisi-tokens is let-bound in wisi-parse-reduce
561 (token (car (nth number wisi-tokens)))
564 ;; Marker one char into token, so indent-line-to
565 ;; inserts space before the mark, not after
566 (when region (copy-marker (1+ (car region)))))
569 (unless (memq class wisi-class-list)
570 (error "%s not in wisi-class-list" class))
574 (if (setq cache (wisi-get-cache (car region)))
575 ;; We are processing a previously set non-terminal; ie generic_formal_part in
577 ;; generic_package_declaration : generic_formal_part package_specification SEMICOLON
578 ;; (wisi-statement-action 1 'block-start 2 'block-middle 3 'statement-end)
580 ;; or simple_statement in
582 ;; statement : label_opt simple_statement
584 ;; override nonterm, class and containing
586 (cl-case (wisi-cache-class cache)
588 (setf (wisi-cache-class cache)
590 ((eq override-start nil)
592 ((memq class '(block-start statement-start)) 'block-start)
595 ((memq override-start '(block-start statement-start)) 'block-start)
597 (t (error "unexpected override-start"))
600 (setf (wisi-cache-class cache) (or override-start class)))
602 (setf (wisi-cache-nonterm cache) $nterm)
603 (setf (wisi-cache-containing cache) first-keyword-mark))
605 ;; else create new cache
606 (with-silent-modifications
612 :nonterm $nterm;; $nterm defined in wisi-semantic-action
614 :last (- (cdr region) (car region))
615 :class (or override-start class)
616 :containing first-keyword-mark)
620 (setq first-item nil)
621 (when (or override-start
622 (memq class '(block-middle block-start statement-start)))
623 (setq override-start nil)
624 (setq first-keyword-mark mark)))
626 (when (eq class 'statement-end)
627 (wisi-set-end wisi-tokens (copy-marker (1+ (car region)))))
630 ;; region is nil when a production is empty; if the first
631 ;; token is a start, override the class on the next token.
632 (when (and first-item
633 (memq class '(block-middle block-start statement-start)))
634 (setq override-start class)))
638 (defun wisi-containing-action (containing-token contained-token)
639 "Set containing marks in all tokens in CONTAINED-TOKEN with null containing mark to marker pointing to CONTAINING-TOKEN.
640 If CONTAINING-TOKEN is empty, the next token number is used."
641 ;; wisi-tokens is is bound in action created by wisi-semantic-action
642 (let* ((containing-region (cddr (nth (1- containing-token) wisi-tokens)))
643 (contained-region (cddr (nth (1- contained-token) wisi-tokens))))
644 (while (not containing-region)
645 ;; containing-token is empty; use next
646 (setq containing-region (cddr (nth containing-token wisi-tokens))))
648 (when contained-region
649 ;; nil when empty production, may not contain any caches
651 (goto-char (cdr contained-region))
652 (let ((cache (wisi-backward-cache))
653 (mark (copy-marker (1+ (car containing-region)))))
656 ;; skip blocks that are already marked
657 (while (and (>= (point) (car contained-region))
658 (markerp (wisi-cache-containing cache)))
659 (goto-char (1- (wisi-cache-containing cache)))
660 (setq cache (wisi-get-cache (point))))
662 (if (or (and (= (car containing-region) (car contained-region))
663 (<= (point) (car contained-region)))
664 (< (point) (car contained-region)))
668 ;; else set mark, loop
669 (setf (wisi-cache-containing cache) mark)
670 (setq cache (wisi-backward-cache)))
673 (defun wisi-motion-action (&rest token-numbers)
674 "Set prev/next marks in all tokens given by TOKEN-NUMBERS.
675 Each TOKEN-NUMBERS is one of:
677 number: the token number; mark that token
679 list (number token_id):
680 list (number (token_id token_id)):
681 mark all tokens with token_id in the nonterminal given by the number."
683 (let (prev-keyword-mark
688 (let ((token-number (pop token-numbers))
692 ((numberp token-number)
693 (setq target-token nil)
694 (setq region (cddr (nth (1- token-number) wisi-tokens)))
696 (setq cache (wisi-get-cache (car region)))
697 (setq mark (copy-marker (1+ (car region))))
699 (when (and prev-keyword-mark
701 (null (wisi-cache-prev cache)))
702 (setf (wisi-cache-prev cache) prev-keyword-mark)
703 (setf (wisi-cache-next prev-cache) mark))
705 (setq prev-keyword-mark mark)
706 (setq prev-cache cache)
709 ((listp token-number)
710 ;; token-number may contain 0, 1, or more token_id; token_id may be a list
711 ;; the corresponding region may be empty
712 ;; there must have been a prev keyword
713 (setq target-token (cadr token-number))
714 (when (not (listp target-token))
715 (setq target-token (list target-token)))
716 (setq token-number (car token-number))
717 (setq region (cddr (nth (1- token-number) wisi-tokens)))
718 (when region ;; not an empty token
719 (goto-char (car region))
720 (while (wisi-forward-find-token target-token (cdr region) t)
721 (setq cache (wisi-get-cache (point)))
722 (setq mark (copy-marker (1+ (point))))
724 (when (null (wisi-cache-prev cache))
725 (setf (wisi-cache-prev cache) prev-keyword-mark)
726 (setf (wisi-cache-next prev-cache) mark)
727 (setq prev-keyword-mark mark)
728 (setq prev-cache cache))
730 (wisi-forward-token);; don't find same token again
735 (error "unexpected token-number %s" token-number))
742 (defun wisi-backward-cache ()
743 "Move point backward to the beginning of the first token preceding point that has a cache.
744 Returns cache, or nil if at beginning of buffer."
746 (setq pos (previous-single-property-change (point) 'wisi-cache))
747 ;; There are three cases:
749 ;; 1) caches separated by non-cache chars: 'if ... then'
750 ;; pos is before 'f', cache is on 'i'
752 ;; 2) caches not separated: ');'
753 ;; pos is before ';', cache is on ';'
755 ;; 3) at bob; pos is nil
759 (setq cache (get-text-property pos 'wisi-cache))
764 (setq cache (get-text-property (1- pos) 'wisi-cache))
765 (goto-char (1- pos))))
767 (goto-char (point-min))
772 (defun wisi-forward-cache ()
773 "Move point forward to the beginning of the first token after point that has a cache.
774 Returns cache, or nil if at end of buffer."
776 (when (get-text-property (point) 'wisi-cache)
777 ;; on a cache; get past it
778 (goto-char (1+ (point))))
780 (setq cache (get-text-property (point) 'wisi-cache))
784 (setq pos (next-single-property-change (point) 'wisi-cache))
788 (setq cache (get-text-property pos 'wisi-cache)))
790 (goto-char (point-max))
796 (defun wisi-forward-find-class (class limit)
797 "Search forward for a token that has a cache with CLASS.
798 Return cache, or nil if at end of buffer.
799 If LIMIT (a buffer position) is reached, throw an error."
800 (let ((cache (wisi-forward-cache)))
801 (while (not (eq class (wisi-cache-class cache)))
802 (setq cache (wisi-forward-cache))
803 (when (>= (point) limit)
804 (error "cache with class %s not found" class)))
807 (defun wisi-forward-find-token (token limit &optional noerror)
808 "Search forward for a token that has a cache with TOKEN.
809 If point is at a matching token, return that token.
810 TOKEN may be a list; stop on any cache that has a member of the list.
811 Return cache, or nil if at end of buffer.
812 If LIMIT (a buffer position) is reached, then if NOERROR is nil, throw an
813 error, if non-nil, return nil."
814 (let ((token-list (cond
815 ((listp token) token)
817 (cache (wisi-get-cache (point)))
821 (memq (wisi-cache-token cache) token-list))))
822 (setq cache (wisi-forward-cache))
823 (when (>= (point) limit)
828 (error "cache with token %s not found" token))))
831 (defun wisi-forward-find-nonterm (nonterm limit)
832 "Search forward for a token that has a cache with NONTERM.
833 NONTERM may be a list; stop on any cache that has a member of the list.
834 Return cache, or nil if at end of buffer.
835 If LIMIT (a buffer position) is reached, throw an error."
836 (let ((nonterm-list (cond
837 ((listp nonterm) nonterm)
839 (cache (wisi-forward-cache)))
840 (while (not (memq (wisi-cache-nonterm cache) nonterm-list))
841 (setq cache (wisi-forward-cache))
842 (when (>= (point) limit)
843 (error "cache with nonterm %s not found" nonterm)))
846 (defun wisi-goto-cache-next (cache)
847 (goto-char (1- (wisi-cache-next cache)))
848 (wisi-get-cache (point))
851 (defun wisi-forward-statement-keyword ()
852 "If not at a cached token, move forward to next
853 cache. Otherwise move to cache-next, or next cache if nil.
855 (wisi-validate-cache (point-max))
856 (let ((cache (wisi-get-cache (point))))
858 (let ((next (wisi-cache-next cache)))
860 (goto-char (1- next))
862 (wisi-forward-cache)))
863 (wisi-forward-cache))
865 (wisi-get-cache (point))
868 (defun wisi-backward-statement-keyword ()
869 "If not at a cached token, move backward to prev
870 cache. Otherwise move to cache-prev, or prev cache if nil."
871 (wisi-validate-cache (point-max))
872 (let ((cache (wisi-get-cache (point))))
874 (let ((prev (wisi-cache-prev cache)))
876 (goto-char (1- prev))
877 (wisi-backward-cache)))
878 (wisi-backward-cache))
881 (defun wisi-goto-containing (cache &optional error)
882 "Move point to containing token for CACHE, return cache at that point."
884 ((markerp (wisi-cache-containing cache))
885 (goto-char (1- (wisi-cache-containing cache)))
886 (wisi-get-cache (point)))
889 (error "already at outermost containing token")))
892 (defun wisi-goto-containing-paren (cache)
893 "Move point to just after the open-paren containing CACHE.
894 Return cache for paren, or nil if no containing paren."
896 (not (eq (wisi-cache-class cache) 'open-paren)))
897 (setq cache (wisi-goto-containing cache)))
902 (defun wisi-goto-start (cache)
903 "Move point to containing ancestor of CACHE that has class block-start or statement-start.
908 (not (memq (wisi-cache-class cache) '(block-start statement-start))))
909 (setq cache (wisi-goto-containing cache)))
913 (defun wisi-goto-end ()
914 "Move point to token at end of statement point is in or before."
916 (wisi-validate-cache (point-max))
917 (let ((cache (or (wisi-get-cache (point))
918 (wisi-forward-cache))))
919 (when (wisi-cache-end cache)
920 ;; nil when cache is statement-end
921 (goto-char (1- (wisi-cache-end cache))))
924 (defun wisi-next-statement-cache (cache)
925 "Move point to CACHE-next, return cache; error if nil."
926 (when (not (markerp (wisi-cache-next cache)))
927 (error "no next statement cache"))
928 (goto-char (1- (wisi-cache-next cache)))
929 (wisi-get-cache (point)))
931 (defun wisi-prev-statement-cache (cache)
932 "Move point to CACHE-next, return cache; error if nil."
933 (when (not (markerp (wisi-cache-prev cache)))
934 (error "no prev statement cache"))
935 (goto-char (1- (wisi-cache-prev cache)))
936 (wisi-get-cache (point)))
940 (defun wisi-comment-indent ()
941 "For `comment-indent-function'. Indent single line comment to
942 the comment on the previous line."
943 ;; This should only be called by comment-indent-new-line or
944 ;; fill-comment-paragraph, so there will be a preceding comment line
945 ;; that we can trust.
948 (if (looking-at comment-start)
950 (error "wisi-comment-indent called after non-comment"))))
952 (defun wisi-indent-current (offset)
953 "Return indentation OFFSET relative to indentation of current line."
954 (+ (current-indentation) offset)
957 (defun wisi-indent-paren (offset)
958 "Return indentation OFFSET relative to preceding open paren."
960 (goto-char (nth 1 (syntax-ppss)))
961 (+ (current-column) offset)))
963 (defun wisi-indent-start (offset cache)
964 "Return indentation of OFFSET relative to containing ancestor
965 of CACHE with class statement-start or block-start."
966 (wisi-goto-start cache)
967 (+ (current-indentation) offset))
969 (defun wisi-indent-statement ()
970 "Indent region given by `wisi-goto-start' on cache at or before point, then wisi-cache-end."
971 ;; force reparse, in case parser got confused
972 (let ((wisi-parse-try t))
973 (wisi-validate-cache (point)))
976 (let ((cache (or (wisi-get-cache (point))
977 (wisi-backward-cache))))
979 ;; can be nil if in header comment
980 (let ((start (progn (wisi-goto-start cache) (point)))
982 (when (wisi-cache-end cache)
983 ;; nil when cache is statement-end
984 (goto-char (1- (wisi-cache-end cache))))
986 (indent-region start end)
990 (defvar-local wisi-indent-calculate-functions nil
991 "Functions to calculate indentation. Each called with point
992 before a token at the beginning of a line (at current
993 indentation); return indentation column for that token, or
994 nil. May move point. Calling stops when first function returns
997 (defvar-local wisi-post-parse-fail-hook
998 "Function to reindent portion of buffer.
999 Called from `wisi-indent-line' when a parse succeeds after
1000 failing; assumes user was editing code that is now syntactically
1001 correct. Must leave point at indentation of current line.")
1003 (defvar-local wisi-indent-failed nil
1004 "Non-nil when wisi-indent-line fails due to parse failing; cleared when indent succeeds.")
1006 (defun wisi-indent-line ()
1007 "Indent current line using the wisi indentation engine."
1010 (let* ((savep (point))
1013 (wisi-validate-cache (point))
1014 (back-to-indentation)
1015 (when (>= (point) savep) (setq savep nil))
1016 (if wisi-parse-failed
1018 ;; parse failed. Assume user is editing; indent to previous line, fix it after parse succeeds
1019 (setq wisi-indent-failed t)
1020 (forward-line -1);; safe at bob
1021 (back-to-indentation)
1024 ;; else parse succeeded
1025 (when wisi-indent-failed
1026 (setq wisi-indent-failed nil)
1027 (run-hooks 'wisi-post-parse-fail-hook))
1028 (with-demoted-errors
1029 (or (run-hook-with-args-until-success 'wisi-indent-calculate-functions) 0))
1032 ;; point was inside line text; leave it there
1033 (save-excursion (indent-line-to indent))
1034 ;; point was before line text; move to start of text
1035 (indent-line-to indent))
1039 (defun wisi-parse-buffer ()
1041 (syntax-propertize (point-max))
1042 (wisi-invalidate-cache)
1043 (wisi-validate-cache (point-max)))
1045 (defun wisi-show-cache ()
1046 "Show cache at point."
1048 (message "%s" (wisi-get-cache (point))))
1050 (defun wisi-show-token ()
1051 "Move forward across one keyword, show token_id."
1053 (let ((token (wisi-forward-token)))
1054 (message "%s" (car token))))
1056 (defun wisi-show-containing-or-previous-cache ()
1058 (let ((cache (wisi-get-cache (point))))
1060 (message "containing %s" (wisi-goto-containing cache t))
1061 (message "previous %s" (wisi-backward-cache)))
1066 (defun wisi-setup (indent-calculate post-parse-fail class-list keyword-table token-table parse-table)
1067 "Set up a buffer for parsing files with wisi."
1068 (setq wisi-class-list class-list)
1069 (setq wisi-string-double-term (car (symbol-value (intern-soft "string-double" token-table))))
1070 (setq wisi-string-single-term (car (symbol-value (intern-soft "string-single" token-table))))
1071 (setq wisi-symbol-term (car (symbol-value (intern-soft "symbol" token-table))))
1073 (setq wisi-punctuation-table (symbol-value (intern-soft "punctuation" token-table)))
1074 (setq wisi-punctuation-table-max-length 0)
1076 (dolist (item wisi-punctuation-table)
1077 (when item ;; default matcher can be nil
1079 ;; check that all chars used in punctuation tokens have punctuation syntax
1080 (mapc (lambda (char)
1081 (when (not (= ?. (char-syntax char)))
1083 (message "in %s, %c does not have punctuation syntax"
1087 (when (< wisi-punctuation-table-max-length (length (cdr item)))
1088 (setq wisi-punctuation-table-max-length (length (cdr item)))))
1091 (error "aborting due to punctuation errors")))
1093 (setq wisi-keyword-table keyword-table)
1094 (setq wisi-parse-table parse-table)
1096 (setq wisi-indent-calculate-functions indent-calculate)
1097 (set (make-local-variable 'indent-line-function) 'wisi-indent-line)
1099 (setq wisi-post-parse-fail-hook post-parse-fail)
1100 (setq wisi-indent-failed nil)
1102 (add-hook 'before-change-functions 'wisi-before-change nil t)
1103 (add-hook 'after-change-functions 'wisi-after-change nil t)
1105 ;; WORKAROUND: sometimes the first time font-lock is run,
1106 ;; syntax-propertize is not run properly, so we run it here
1107 (syntax-propertize (point-max))
1109 (wisi-invalidate-cache)
1111 ;; FIXME: debug counter
1112 (setq wisi-change-jit-lock-mode 0)
1116 ;;; wisi.el ends here