;;; wisent-python.el --- Semantic support for Python
-;; Copyright (C) 2002, 2004, 2006-2011 Free Software Foundation, Inc.
+;; Copyright (C) 2002, 2004, 2006-2015 Free Software Foundation, Inc.
;; Author: Richard Kim <emacs18@gmail.com>
;; Maintainer: Richard Kim <emacs18@gmail.com>
;;; Code:
+(require 'rx)
+
+;; Try to load python support, but fail silently since it is only used
+;; for optional functionality
+(require 'python nil t)
+
(require 'semantic/wisent)
(require 'semantic/wisent/python-wy)
+(require 'semantic/find)
(require 'semantic/dep)
(require 'semantic/ctxt)
+(require 'semantic/format)
+
+(eval-when-compile
+ (require 'cl))
+
+;;; Customization
+;;
+(defun semantic-python-get-system-include-path ()
+ "Evaluate some Python code that determines the system include path."
+ (delq nil
+ (mapcar
+ (lambda (dir)
+ (when (file-directory-p dir)
+ dir))
+ (split-string
+ (python-shell-internal-send-string
+ "import sys;print ('\\n'.join(sys.path))")
+ "\n" t))))
+
+(defcustom-mode-local-semantic-dependency-system-include-path
+ python-mode semantic-python-dependency-system-include-path
+ (when (and (featurep 'python)
+ ;; python-mode and batch somehow often hangs.
+ (not noninteractive))
+ (semantic-python-get-system-include-path))
+ "The system include path used by Python language.")
\f
;;; Lexical analysis
;;
;; Python strings are delimited by either single quotes or double
-;; quotes, e.g., "I'm a string" and 'I too am s string'.
+;; quotes, e.g., "I'm a string" and 'I too am a string'.
;; In addition a string can have either a 'r' and/or 'u' prefix.
;; The 'r' prefix means raw, i.e., normal backslash substitutions are
;; to be suppressed. For example, r"01\n34" is a string with six
;; characters 0, 1, \, n, 3 and 4. The 'u' prefix means the following
-;; string is a unicode.
-(defconst wisent-python-string-re
- (concat (regexp-opt '("r" "u" "ur" "R" "U" "UR" "Ur" "uR") t)
- "?['\"]")
+;; string is Unicode.
+(defconst wisent-python-string-start-re "[uU]?[rR]?['\"]"
"Regexp matching beginning of a Python string.")
+(defconst wisent-python-string-re
+ (rx
+ (opt (any "uU")) (opt (any "rR"))
+ (or
+ ;; Triple-quoted string using apostrophes
+ (: "'''" (zero-or-more (or "\\'"
+ (not (any "'"))
+ (: (repeat 1 2 "'") (not (any "'")))))
+ "'''")
+ ;; String using apostrophes
+ (: "'" (zero-or-more (or "\\'"
+ (not (any "'"))))
+ "'")
+ ;; Triple-quoted string using quotation marks.
+ (: "\"\"\"" (zero-or-more (or "\\\""
+ (not (any "\""))
+ (: (repeat 1 2 "\"") (not (any "\"")))))
+ "\"\"\"")
+ ;; String using quotation marks.
+ (: "\"" (zero-or-more (or "\\\""
+ (not (any "\""))))
+ "\"")))
+ "Regexp matching a complete Python string.")
+
(defvar wisent-python-EXPANDING-block nil
"Non-nil when expanding a paren block for Python lexical analyzer.")
(defsubst wisent-python-forward-string ()
"Move point at the end of the Python string at point."
- (when (looking-at wisent-python-string-re)
- ;; skip the prefix
- (and (match-end 1) (goto-char (match-end 1)))
- ;; skip the quoted part
- (cond
- ((looking-at "\"\"\"[^\"]")
- (search-forward "\"\"\"" nil nil 2))
- ((looking-at "'''[^']")
- (search-forward "'''" nil nil 2))
- ((forward-sexp 1)))))
+ (if (looking-at wisent-python-string-re)
+ (let ((start (match-beginning 0))
+ (end (match-end 0)))
+ ;; Incomplete triple-quoted string gets matched instead as a
+ ;; complete single quoted string. (This special case would be
+ ;; unnecessary if Emacs regular expressions had negative
+ ;; look-ahead assertions.)
+ (when (and (= (- end start) 2)
+ (looking-at "\"\\{3\\}\\|'\\{3\\}"))
+ (error "unterminated syntax"))
+ (goto-char end))
+ (error "unterminated syntax")))
+
+(defun wisent-python-forward-balanced-expression ()
+ "Move point to the end of the balanced expression at point.
+Here 'balanced expression' means anything matched by Emacs'
+open/close parenthesis syntax classes. We can't use forward-sexp
+for this because that Emacs built-in can't parse Python's
+triple-quoted string syntax."
+ (let ((end-char (cdr (syntax-after (point)))))
+ (forward-char 1)
+ (while (not (or (eobp) (eq (char-after (point)) end-char)))
+ (cond
+ ;; Skip over python strings.
+ ((looking-at wisent-python-string-start-re)
+ (wisent-python-forward-string))
+ ;; At a comment start just goto end of line.
+ ((looking-at "\\s<")
+ (end-of-line))
+ ;; Skip over balanced expressions.
+ ((looking-at "\\s(")
+ (wisent-python-forward-balanced-expression))
+ ;; Skip over white space, word, symbol, punctuation, paired
+ ;; delimiter (backquote) characters, line continuation, and end
+ ;; of comment characters (AKA newline characters in Python).
+ ((zerop (skip-syntax-forward "-w_.$\\>"))
+ (error "can't figure out how to go forward from here"))))
+ ;; Skip closing character. As a last resort this should raise an
+ ;; error if we hit EOB before we find our closing character..
+ (forward-char 1)))
(defun wisent-python-forward-line ()
"Move point to the beginning of the next logical line.
(progn
(cond
;; Skip over python strings.
- ((looking-at wisent-python-string-re)
+ ((looking-at wisent-python-string-start-re)
(wisent-python-forward-string))
;; At a comment start just goto end of line.
((looking-at "\\s<")
(end-of-line))
- ;; Skip over generic lists and strings.
- ((looking-at "\\(\\s(\\|\\s\"\\)")
- (forward-sexp 1))
+ ;; Skip over balanced expressions.
+ ((looking-at "\\s(")
+ (wisent-python-forward-balanced-expression))
;; At the explicit line continuation character
;; (backslash) move to next line.
((looking-at "\\s\\")
(defun wisent-python-forward-line-skip-indented ()
"Move point to the next logical line, skipping indented lines.
-That is the next line whose indentation is less than or equal to the
-identation of the current line."
+That is the next line whose indentation is less than or equal to
+the indentation of the current line."
(let ((indent (current-indentation)))
(while (progn (wisent-python-forward-line)
(and (not (eobp))
;; Loop lexer to handle tokens in current line.
t)
;; Indentation decreased
- (t
- ;; Pop items from indentation stack
- (while (< curr-indent last-indent)
- (pop wisent-python-indent-stack)
- (setq semantic-lex-current-depth (1- semantic-lex-current-depth)
- last-indent (car wisent-python-indent-stack))
- (semantic-lex-push-token
- (semantic-lex-token 'DEDENT last-pos (point))))
+ ((progn
+ ;; Pop items from indentation stack
+ (while (< curr-indent last-indent)
+ (pop wisent-python-indent-stack)
+ (setq semantic-lex-current-depth (1- semantic-lex-current-depth)
+ last-indent (car wisent-python-indent-stack))
+ (semantic-lex-push-token
+ (semantic-lex-token 'DEDENT last-pos (point))))
+ (= last-pos (point)))
;; If pos did not change, then we must return nil so that
;; other lexical analyzers can be run.
- (/= last-pos (point))))))
+ nil))))
;; All the work was done in the above analyzer matching condition.
)
(define-lex-regex-analyzer wisent-python-lex-string
"Detect and create python string tokens."
- wisent-python-string-re
+ wisent-python-string-start-re
(semantic-lex-push-token
(semantic-lex-token
'STRING_LITERAL
semantic-lex-ignore-comments
;; Signal error on unhandled syntax.
semantic-lex-default-action)
+
+\f
+;;; Parsing
+;;
+
+(defun wisent-python-reconstitute-function-tag (tag suite)
+ "Move a docstring from TAG's members into its :documentation attribute.
+Set attributes for constructors, special, private and static methods."
+ ;; Analyze first statement to see whether it is a documentation
+ ;; string.
+ (let ((first-statement (car suite)))
+ (when (semantic-python-docstring-p first-statement)
+ (semantic-tag-put-attribute
+ tag :documentation
+ (semantic-python-extract-docstring first-statement))))
+
+ ;; TODO HACK: we try to identify methods using the following
+ ;; heuristic:
+ ;; + at least one argument
+ ;; + first argument is self
+ (when (and (> (length (semantic-tag-function-arguments tag)) 0)
+ (string= (semantic-tag-name
+ (first (semantic-tag-function-arguments tag)))
+ "self"))
+ (semantic-tag-put-attribute tag :parent "dummy"))
+
+ ;; Identify constructors, special and private functions
+ (cond
+ ;; TODO only valid when the function resides inside a class
+ ((string= (semantic-tag-name tag) "__init__")
+ (semantic-tag-put-attribute tag :constructor-flag t)
+ (semantic-tag-put-attribute tag :suite suite))
+
+ ((semantic-python-special-p tag)
+ (semantic-tag-put-attribute tag :special-flag t))
+
+ ((semantic-python-private-p tag)
+ (semantic-tag-put-attribute tag :protection "private")))
+
+ ;; If there is a staticmethod decorator, add a static typemodifier
+ ;; for the function.
+ (when (semantic-find-tags-by-name
+ "staticmethod"
+ (semantic-tag-get-attribute tag :decorators))
+ (semantic-tag-put-attribute
+ tag :typemodifiers
+ (cons "static"
+ (semantic-tag-get-attribute tag :typemodifiers))))
+
+ ;; TODO
+ ;; + check for decorators classmethod
+ ;; + check for operators
+ tag)
+
+(defun wisent-python-reconstitute-class-tag (tag)
+ "Move a docstring from TAG's members into its :documentation attribute."
+ ;; The first member of TAG may be a documentation string. If that is
+ ;; the case, remove of it from the members list and stick its
+ ;; content into the :documentation attribute.
+ (let ((first-member (car (semantic-tag-type-members tag))))
+ (when (semantic-python-docstring-p first-member)
+ (semantic-tag-put-attribute
+ tag :members
+ (cdr (semantic-tag-type-members tag)))
+ (semantic-tag-put-attribute
+ tag :documentation
+ (semantic-python-extract-docstring first-member))))
+
+ ;; Try to find the constructor, determine the name of the instance
+ ;; parameter, find assignments to instance variables and add
+ ;; corresponding variable tags to the list of members.
+ (dolist (member (semantic-tag-type-members tag))
+ (when (semantic-tag-function-constructor-p member)
+ (let ((self (semantic-tag-name
+ (car (semantic-tag-function-arguments member)))))
+ (dolist (statement (semantic-tag-get-attribute member :suite))
+ (when (semantic-python-instance-variable-p statement self)
+ (let ((variable (semantic-tag-clone
+ statement
+ (substring (semantic-tag-name statement) 5)))
+ (members (semantic-tag-get-attribute tag :members)))
+ (when (semantic-python-private-p variable)
+ (semantic-tag-put-attribute variable :protection "private"))
+ (setcdr (last members) (list variable))))))))
+
+ ;; TODO remove the :suite attribute
+ tag)
+
+(defun semantic-python-expand-tag (tag)
+ "Expand compound declarations found in TAG into separate tags.
+TAG contains compound declaration if the NAME part of the tag is
+a list. In python, this can happen with `import' statements."
+ (let ((class (semantic-tag-class tag))
+ (elts (semantic-tag-name tag))
+ (expand nil))
+ (cond
+ ((and (eq class 'include) (listp elts))
+ (dolist (E elts)
+ (setq expand (cons (semantic-tag-clone tag E) expand)))
+ (setq expand (nreverse expand)))
+ )))
+
+
\f
;;; Overridden Semantic API.
;;
+
(define-mode-local-override semantic-lex python-mode
(start end &optional depth length)
"Lexically analyze Python code in current buffer.
To be implemented for Python! For now just return nil."
nil)
-(defcustom-mode-local-semantic-dependency-system-include-path
- python-mode semantic-python-dependency-system-include-path
- nil
- "The system include path used by Python language.")
+;; Adapted from the semantic Java support by Andrey Torba
+(define-mode-local-override semantic-tag-include-filename python-mode (tag)
+ "Return a suitable path for (some) Python imports."
+ (let ((name (semantic-tag-name tag)))
+ (concat (mapconcat 'identity (split-string name "\\.") "/") ".py")))
+
+;; Override ctxt-current-function/assignment defaults, since they do
+;; not work properly with Python code, even leading to endless loops
+;; (see bug #xxxxx).
+(define-mode-local-override semantic-ctxt-current-function python-mode (&optional point)
+ "Return the current function call the cursor is in at POINT.
+The function returned is the one accepting the arguments that
+the cursor is currently in. It will not return function symbol if the
+cursor is on the text representing that function."
+ nil)
+
+(define-mode-local-override semantic-ctxt-current-assignment python-mode (&optional point)
+ "Return the current assignment near the cursor at POINT.
+Return a list as per `semantic-ctxt-current-symbol'.
+Return nil if there is nothing relevant."
+ nil)
+
+;;; Tag Formatting
+;;
+(define-mode-local-override semantic-format-tag-abbreviate python-mode (tag &optional parent color)
+ "Format an abbreviated tag for python.
+Shortens 'code' tags, but passes through for others."
+ (cond ((semantic-tag-of-class-p tag 'code)
+ ;; Just take the first line.
+ (let ((name (semantic-tag-name tag)))
+ (when (string-match "\n" name)
+ (setq name (substring name 0 (match-beginning 0))))
+ name))
+ (t
+ (semantic-format-tag-abbreviate-default tag parent color))))
;;; Enable Semantic in `python-mode'.
;;
"Setup buffer for parse."
(wisent-python-wy--install-parser)
(set (make-local-variable 'parse-sexp-ignore-comments) t)
+ ;; Give python modes the possibility to overwrite this:
+ (if (not comment-start-skip)
+ (set (make-local-variable 'comment-start-skip) "#+\\s-*"))
(setq
- ;; Character used to separation a parent/child relationship
+ ;; Character used to separation a parent/child relationship
semantic-type-relation-separator-character '(".")
semantic-command-separation-character ";"
- ;; The following is no more necessary as semantic-lex is overriden
- ;; in python-mode.
- ;; semantic-lex-analyzer 'wisent-python-lexer
+ ;; Parsing
+ semantic-tag-expand-function 'semantic-python-expand-tag
;; Semantic to take over from the one provided by python.
;; The python one, if it uses the senator advice, will hang
(define-child-mode python-2-mode python-mode "Python 2 mode")
(define-child-mode python-3-mode python-mode "Python 3 mode")
+\f
+;;; Utility functions
+;;
+
+(defun semantic-python-special-p (tag)
+ "Return non-nil if the name of TAG is a special identifier of
+the form __NAME__. "
+ (string-match
+ (rx (seq string-start "__" (1+ (syntax symbol)) "__" string-end))
+ (semantic-tag-name tag)))
+
+(defun semantic-python-private-p (tag)
+ "Return non-nil if the name of TAG follows the convention _NAME
+for private names."
+ (string-match
+ (rx (seq string-start "_" (0+ (syntax symbol)) string-end))
+ (semantic-tag-name tag)))
+
+(defun semantic-python-instance-variable-p (tag &optional self)
+ "Return non-nil if TAG is an instance variable of the instance
+SELF or the instance name \"self\" if SELF is nil."
+ (when (semantic-tag-of-class-p tag 'variable)
+ (let ((name (semantic-tag-name tag)))
+ (when (string-match
+ (rx-to-string
+ `(seq string-start ,(or self "self") "."))
+ name)
+ (not (string-match "\\." (substring name 5)))))))
+
+(defun semantic-python-docstring-p (tag)
+ "Return non-nil, when TAG is a Python documentation string."
+ ;; TAG is considered to be a documentation string if the first
+ ;; member is of class 'code and its name looks like a documentation
+ ;; string.
+ (let ((class (semantic-tag-class tag))
+ (name (semantic-tag-name tag)))
+ (and (eq class 'code)
+ (string-match
+ (rx (seq string-start "\"\"\"" (0+ anything) "\"\"\"" string-end))
+ name))))
+
+(defun semantic-python-extract-docstring (tag)
+ "Return the Python documentation string contained in TAG."
+ ;; Strip leading and trailing """
+ (let ((name (semantic-tag-name tag)))
+ (substring name 3 -3)))
+
\f
;;; Test
;;
+
(defun wisent-python-lex-buffer ()
"Run `wisent-python-lexer' on current buffer."
(interactive)