X-Git-Url: https://code.delx.au/gnu-emacs/blobdiff_plain/325c5543035b411ae79839dda47bbbbde838d36b..604f6568312aef8287d7a3ac1dbeb71577089bec:/lisp/progmodes/python.el diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el index 42272a9d55..2d22bb2ce8 100644 --- a/lisp/progmodes/python.el +++ b/lisp/progmodes/python.el @@ -1,10 +1,11 @@ ;;; python.el --- Python's flying circus support for Emacs -*- lexical-binding: t -*- -;; Copyright (C) 2003-2015 Free Software Foundation, Inc. +;; Copyright (C) 2003-2016 Free Software Foundation, Inc. -;; Author: Fabián E. Gallina +;; Author: Fabián E. Gallina ;; URL: https://github.com/fgallina/python.el -;; Version: 0.24.5 +;; Version: 0.25.1 +;; Package-Requires: ((emacs "24.1") (cl-lib "1.0")) ;; Maintainer: emacs-devel@gnu.org ;; Created: Jul 2010 ;; Keywords: languages @@ -180,6 +181,12 @@ ;; shell so that relative imports work properly using the ;; `python-shell-package-enable' command. +;; Shell remote support: remote Python shells are started with the +;; correct environment for files opened remotely through tramp, also +;; respecting dir-local variables provided `enable-remote-dir-locals' +;; is non-nil. The logic for this is transparently handled by the +;; `python-shell-with-environment' macro. + ;; Shell syntax highlighting: when enabled current input in shell is ;; highlighted. The variable `python-shell-font-lock-enable' controls ;; activation of this feature globally when shells are started. @@ -255,6 +262,7 @@ (require 'cl-lib) (require 'comint) (require 'json) +(require 'tramp-sh) ;; Avoid compiler warnings (defvar view-return-to-alist) @@ -265,7 +273,7 @@ (autoload 'help-function-arglist "help-fns") ;;;###autoload -(add-to-list 'auto-mode-alist (cons (purecopy "\\.py\\'") 'python-mode)) +(add-to-list 'auto-mode-alist (cons (purecopy "\\.pyw?\\'") 'python-mode)) ;;;###autoload (add-to-list 'interpreter-mode-alist (cons (purecopy "python[0-9.]*") 'python-mode)) @@ -284,6 +292,7 @@ (define-key map [remap backward-sentence] 'python-nav-backward-block) (define-key map [remap forward-sentence] 'python-nav-forward-block) (define-key map [remap backward-up-list] 'python-nav-backward-up-list) + (define-key map [remap mark-defun] 'python-mark-defun) (define-key map "\C-c\C-j" 'imenu) ;; Indent specific (define-key map "\177" 'python-indent-dedent-line-backspace) @@ -359,11 +368,14 @@ ;;; Python specialized rx -(eval-when-compile +(eval-and-compile (defconst python-rx-constituents `((block-start . ,(rx symbol-start (or "def" "class" "if" "elif" "else" "try" - "except" "finally" "for" "while" "with") + "except" "finally" "for" "while" "with" + ;; Python 3.5+ PEP492 + (and "async" (+ space) + (or "def" "for" "with"))) symbol-end)) (dedenter . ,(rx symbol-start (or "elif" "else" "except" "finally") @@ -374,7 +386,11 @@ symbol-end)) (decorator . ,(rx line-start (* space) ?@ (any letter ?_) (* (any word ?_)))) - (defun . ,(rx symbol-start (or "def" "class") symbol-end)) + (defun . ,(rx symbol-start + (or "def" "class" + ;; Python 3.5+ PEP492 + (and "async" (+ space) "def")) + symbol-end)) (if-name-main . ,(rx line-start "if" (+ space) "__name__" (+ space) "==" (+ space) (any ?' ?\") "__main__" (any ?' ?\") @@ -430,7 +446,7 @@ This variant of `rx' supports common Python named REGEXPS." ;;; Font-lock and syntax -(eval-when-compile +(eval-and-compile (defun python-syntax--context-compiler-macro (form type &optional syntax-ppss) (pcase type (`'comment @@ -464,13 +480,13 @@ The type returned can be `comment', `string' or `paren'." ((nth 1 ppss) 'paren)))) (defsubst python-syntax-comment-or-string-p (&optional ppss) - "Return non-nil if PPSS is inside 'comment or 'string." + "Return non-nil if PPSS is inside comment or string." (nth 8 (or ppss (syntax-ppss)))) (defsubst python-syntax-closing-paren-p () "Return non-nil if char after point is a closing paren." - (= (syntax-class (syntax-after (point))) - (syntax-class (string-to-syntax ")")))) + (eql (syntax-class (syntax-after (point))) + (syntax-class (string-to-syntax ")")))) (define-obsolete-function-alias 'python-info-ppss-context #'python-syntax-context "24.3") @@ -482,19 +498,10 @@ The type returned can be `comment', `string' or `paren'." 'python-info-ppss-comment-or-string-p #'python-syntax-comment-or-string-p "24.3") -(defun python-docstring-at-p (pos) - "Check to see if there is a docstring at POS." - (save-excursion - (goto-char pos) - (if (looking-at-p "'''\\|\"\"\"") - (progn - (python-nav-backward-statement) - (looking-at "\\`\\|class \\|def ")) - nil))) - (defun python-font-lock-syntactic-face-function (state) + "Return syntactic face given STATE." (if (nth 3 state) - (if (python-docstring-at-p (nth 8 state)) + (if (python-info-docstring-p state) font-lock-doc-face font-lock-string-face) font-lock-comment-face)) @@ -515,6 +522,9 @@ The type returned can be `comment', `string' or `paren'." ;; fontified like that in order to keep font-lock consistent between ;; Python versions. "nonlocal" + ;; Python 3.5+ PEP492 + (and "async" (+ space) (or "def" "for" "with")) + "await" ;; Extra: "self") symbol-end) @@ -608,6 +618,11 @@ The type returned can be `comment', `string' or `paren'." ((python-rx string-delimiter) (0 (ignore (python-syntax-stringify)))))) +(defconst python--prettify-symbols-alist + '(("lambda" . ?λ) + ("and" . ?∧) + ("or" . ?∨))) + (defsubst python-syntax-count-quotes (quote-char &optional point limit) "Count number of quotes around point (max is 3). QUOTE-CHAR is the quote char to count. Optional argument POINT is @@ -696,6 +711,13 @@ It makes underscores and dots word constituent chars.") :group 'python :safe 'booleanp) +(defcustom python-indent-guess-indent-offset-verbose t + "Non-nil means to emit a warning when indentation guessing fails." + :version "25.1" + :type 'boolean + :group 'python + :safe' booleanp) + (defcustom python-indent-trigger-commands '(indent-for-tab-command yas-expand yas/expand) "Commands that might trigger a `python-indent-line' call." @@ -766,8 +788,9 @@ work on `python-indent-calculate-indentation' instead." (current-indentation)))) (if (and indentation (not (zerop indentation))) (set (make-local-variable 'python-indent-offset) indentation) - (message "Can't guess python-indent-offset, using defaults: %s" - python-indent-offset))))))) + (when python-indent-guess-indent-offset-verbose + (message "Can't guess python-indent-offset, using defaults: %s" + python-indent-offset)))))))) (defun python-indent-context () "Get information about the current indentation context. @@ -846,7 +869,9 @@ keyword ;; Inside a string. ((let ((start (python-syntax-context 'string ppss))) (when start - (cons :inside-string start)))) + (cons (if (python-info-docstring-p) + :inside-docstring + :inside-string) start)))) ;; Inside a paren. ((let* ((start (python-syntax-context 'paren ppss)) (starts-in-newline @@ -958,18 +983,20 @@ keyword ((save-excursion (back-to-indentation) (skip-chars-backward " \t\n") - (python-nav-beginning-of-statement) - (cons - (cond ((python-info-current-line-comment-p) - :after-comment) - ((save-excursion - (goto-char (line-end-position)) - (python-util-forward-comment -1) - (python-nav-beginning-of-statement) - (looking-at (python-rx block-ender))) - :after-block-end) - (t :after-line)) - (point)))))))) + (if (bobp) + (cons :no-indent 0) + (python-nav-beginning-of-statement) + (cons + (cond ((python-info-current-line-comment-p) + :after-comment) + ((save-excursion + (goto-char (line-end-position)) + (python-util-forward-comment -1) + (python-nav-beginning-of-statement) + (looking-at (python-rx block-ender))) + :after-block-end) + (t :after-line)) + (point))))))))) (defun python-indent--calculate-indentation () "Internal implementation of `python-indent-calculate-indentation'. @@ -991,6 +1018,12 @@ possibilities can be narrowed to specific indentation points." ;; Copy previous indentation. (goto-char start) (current-indentation)) + (`(:inside-docstring . ,start) + (let* ((line-indentation (current-indentation)) + (base-indent (progn + (goto-char start) + (current-indentation)))) + (max line-indentation base-indent))) (`(,(or :after-block-start :after-backslash-first-line :inside-paren-newline-start) . ,start) @@ -1031,15 +1064,9 @@ integers. Levels are returned in ascending order, and in the case INDENTATION is a list, this order is enforced." (if (listp indentation) (sort (copy-sequence indentation) #'<) - (let* ((remainder (% indentation python-indent-offset)) - (steps (/ (- indentation remainder) python-indent-offset)) - (levels (mapcar (lambda (step) - (* python-indent-offset step)) - (number-sequence steps 0 -1)))) - (reverse - (if (not (zerop remainder)) - (cons indentation levels) - levels))))) + (nconc (number-sequence 0 (1- indentation) + python-indent-offset) + (list indentation)))) (defun python-indent--previous-level (levels indentation) "Return previous level from LEVELS relative to INDENTATION." @@ -1140,14 +1167,15 @@ Called from a program, START and END specify the region to indent." (not line-is-comment-p)) (python-info-current-line-empty-p))))) ;; Don't mess with strings, unless it's the - ;; enclosing set of quotes. + ;; enclosing set of quotes or a docstring. (or (not (python-syntax-context 'string)) (eq (syntax-after (+ (1- (point)) (current-indentation) (python-syntax-count-quotes (char-after) (point)))) - (string-to-syntax "|"))) + (string-to-syntax "|")) + (python-info-docstring-p)) ;; Skip if current line is a block start, a ;; dedenter or block ender. (save-excursion @@ -1247,6 +1275,21 @@ the line will be re-indented automatically if needed." ;; Reindent region if this is a multiline statement (python-indent-region dedenter-pos current-pos))))))))) + +;;; Mark + +(defun python-mark-defun (&optional allow-extend) + "Put mark at end of this defun, point at beginning. +The defun marked is the one that contains point or follows point. + +Interactively (or with ALLOW-EXTEND non-nil), if this command is +repeated or (in Transient Mark mode) if the mark is active, it +marks the next defun after the ones already marked." + (interactive "p") + (when (python-info-looking-at-beginning-of-defun) + (end-of-line 1)) + (mark-defun allow-extend)) + ;;; Navigation @@ -1573,11 +1616,13 @@ forward only one sexp, else move backwards." (while (and (funcall search-fn paren-regexp nil t) (python-syntax-context 'paren))))))) -(defun python-nav--forward-sexp (&optional dir safe) +(defun python-nav--forward-sexp (&optional dir safe skip-parens-p) "Move to forward sexp. With positive optional argument DIR direction move forward, else backwards. When optional argument SAFE is non-nil do not throw -errors when at end of sexp, skip it instead." +errors when at end of sexp, skip it instead. With optional +argument SKIP-PARENS-P force sexp motion to ignore parenthesized +expressions when looking at them in either direction." (setq dir (or dir 1)) (unless (= dir 0) (let* ((forward-p (if (> dir 0) @@ -1589,11 +1634,13 @@ errors when at end of sexp, skip it instead." ;; Inside of a string, get out of it. (let ((forward-sexp-function)) (forward-sexp dir))) - ((or (eq context-type 'paren) - (and forward-p (looking-at (python-rx open-paren))) - (and (not forward-p) - (eq (syntax-class (syntax-after (1- (point)))) - (car (string-to-syntax ")"))))) + ((and (not skip-parens-p) + (or (eq context-type 'paren) + (if forward-p + (eq (syntax-class (syntax-after (point))) + (car (string-to-syntax "("))) + (eq (syntax-class (syntax-after (1- (point)))) + (car (string-to-syntax ")")))))) ;; Inside a paren or looking at it, lisp knows what to do. (if safe (python-nav--lisp-forward-sexp-safe dir) @@ -1629,7 +1676,7 @@ errors when at end of sexp, skip it instead." (cond ((and (not (eobp)) (python-info-current-line-empty-p)) (python-util-forward-comment dir) - (python-nav--forward-sexp dir)) + (python-nav--forward-sexp dir safe skip-parens-p)) ((eq context 'block-start) (python-nav-end-of-block)) ((eq context 'statement-start) @@ -1649,7 +1696,7 @@ errors when at end of sexp, skip it instead." (cond ((and (not (bobp)) (python-info-current-line-empty-p)) (python-util-forward-comment dir) - (python-nav--forward-sexp dir)) + (python-nav--forward-sexp dir safe skip-parens-p)) ((eq context 'block-end) (python-nav-beginning-of-block)) ((eq context 'statement-end) @@ -1667,47 +1714,69 @@ errors when at end of sexp, skip it instead." (python-nav-beginning-of-statement)) (t (goto-char next-sexp-pos)))))))))) -(defun python-nav-forward-sexp (&optional arg) +(defun python-nav-forward-sexp (&optional arg safe skip-parens-p) "Move forward across expressions. With ARG, do it that many times. Negative arg -N means move -backward N times." +backward N times. When optional argument SAFE is non-nil do not +throw errors when at end of sexp, skip it instead. With optional +argument SKIP-PARENS-P force sexp motion to ignore parenthesized +expressions when looking at them in either direction (forced to t +in interactive calls)." (interactive "^p") (or arg (setq arg 1)) + ;; Do not follow parens on interactive calls. This hack to detect + ;; if the function was called interactively copes with the way + ;; `forward-sexp' works by calling `forward-sexp-function', losing + ;; interactive detection by checking `current-prefix-arg'. The + ;; reason to make this distinction is that lisp functions like + ;; `blink-matching-open' get confused causing issues like the one in + ;; Bug#16191. With this approach the user gets a symmetric behavior + ;; when working interactively while called functions expecting + ;; paren-based sexp motion work just fine. + (or + skip-parens-p + (setq skip-parens-p + (memq real-this-command + (list + #'forward-sexp #'backward-sexp + #'python-nav-forward-sexp #'python-nav-backward-sexp + #'python-nav-forward-sexp-safe #'python-nav-backward-sexp)))) (while (> arg 0) - (python-nav--forward-sexp 1) + (python-nav--forward-sexp 1 safe skip-parens-p) (setq arg (1- arg))) (while (< arg 0) - (python-nav--forward-sexp -1) + (python-nav--forward-sexp -1 safe skip-parens-p) (setq arg (1+ arg)))) -(defun python-nav-backward-sexp (&optional arg) +(defun python-nav-backward-sexp (&optional arg safe skip-parens-p) "Move backward across expressions. With ARG, do it that many times. Negative arg -N means move -forward N times." +forward N times. When optional argument SAFE is non-nil do not +throw errors when at end of sexp, skip it instead. With optional +argument SKIP-PARENS-P force sexp motion to ignore parenthesized +expressions when looking at them in either direction (forced to t +in interactive calls)." (interactive "^p") (or arg (setq arg 1)) - (python-nav-forward-sexp (- arg))) + (python-nav-forward-sexp (- arg) safe skip-parens-p)) -(defun python-nav-forward-sexp-safe (&optional arg) +(defun python-nav-forward-sexp-safe (&optional arg skip-parens-p) "Move forward safely across expressions. With ARG, do it that many times. Negative arg -N means move -backward N times." +backward N times. With optional argument SKIP-PARENS-P force +sexp motion to ignore parenthesized expressions when looking at +them in either direction (forced to t in interactive calls)." (interactive "^p") - (or arg (setq arg 1)) - (while (> arg 0) - (python-nav--forward-sexp 1 t) - (setq arg (1- arg))) - (while (< arg 0) - (python-nav--forward-sexp -1 t) - (setq arg (1+ arg)))) + (python-nav-forward-sexp arg t skip-parens-p)) -(defun python-nav-backward-sexp-safe (&optional arg) +(defun python-nav-backward-sexp-safe (&optional arg skip-parens-p) "Move backward safely across expressions. With ARG, do it that many times. Negative arg -N means move -forward N times." +forward N times. With optional argument SKIP-PARENS-P force sexp +motion to ignore parenthesized expressions when looking at them in +either direction (forced to t in interactive calls)." (interactive "^p") - (or arg (setq arg 1)) - (python-nav-forward-sexp-safe (- arg))) + (python-nav-backward-sexp arg t skip-parens-p)) (defun python-nav--up-list (&optional dir) "Internal implementation of `python-nav-up-list'. @@ -1772,7 +1841,7 @@ This command assumes point is not in a string or comment." (defun python-nav-if-name-main () "Move point at the beginning the __main__ block. -When \"if __name__ == '__main__':\" is found returns its +When \"if __name__ == \\='__main__\\=':\" is found returns its position, else returns nil." (interactive) (let ((point (point)) @@ -1893,53 +1962,61 @@ Python shell. See commentary for details." :safe 'booleanp) (defcustom python-shell-process-environment nil - "List of environment variables for Python shell. -This variable follows the same rules as `process-environment' -since it merges with it before the process creation routines are -called. When this variable is nil, the Python shell is run with -the default `process-environment'." + "List of overridden environment variables for subprocesses to inherit. +Each element should be a string of the form ENVVARNAME=VALUE. +When this variable is non-nil, values are exported into the +process environment before starting it. Any variables already +present in the current environment are superseded by variables +set here." :type '(repeat string) - :group 'python - :safe 'listp) + :group 'python) (defcustom python-shell-extra-pythonpaths nil "List of extra pythonpaths for Python shell. -The values of this variable are added to the existing value of -PYTHONPATH in the `process-environment' variable." +When this variable is non-nil, values added at the beginning of +the PYTHONPATH before starting processes. Any values present +here that already exists in PYTHONPATH are moved to the beginning +of the list so that they are prioritized when looking for +modules." :type '(repeat string) - :group 'python - :safe 'listp) + :group 'python) (defcustom python-shell-exec-path nil - "List of path to search for binaries. -This variable follows the same rules as `exec-path' since it -merges with it before the process creation routines are called. -When this variable is nil, the Python shell is run with the -default `exec-path'." + "List of paths for searching executables. +When this variable is non-nil, values added at the beginning of +the PATH before starting processes. Any values present here that +already exists in PATH are moved to the beginning of the list so +that they are prioritized when looking for executables." :type '(repeat string) - :group 'python - :safe 'listp) + :group 'python) + +(defcustom python-shell-remote-exec-path nil + "List of paths to be ensured remotely for searching executables. +When this variable is non-nil, values are exported into remote +hosts PATH before starting processes. Values defined in +`python-shell-exec-path' will take precedence to paths defined +here. Normally you wont use this variable directly unless you +plan to ensure a particular set of paths to all Python shell +executed through tramp connections." + :version "25.1" + :type '(repeat string) + :group 'python) (defcustom python-shell-virtualenv-root nil "Path to virtualenv root. -This variable, when set to a string, makes the values stored in -`python-shell-process-environment' and `python-shell-exec-path' -to be modified properly so shells are started with the specified +This variable, when set to a string, makes the environment to be +modified such that shells are started within the specified virtualenv." :type '(choice (const nil) string) - :group 'python - :safe 'stringp) + :group 'python) (define-obsolete-variable-alias 'python-shell-virtualenv-path 'python-shell-virtualenv-root "25.1") -(defcustom python-shell-setup-codes '(python-shell-completion-setup-code - python-ffap-setup-code - python-eldoc-setup-code) +(defcustom python-shell-setup-codes nil "List of code run by `python-shell-send-setup-codes'." :type '(repeat symbol) - :group 'python - :safe 'listp) + :group 'python) (defcustom python-shell-compilation-regexp-alist `((,(rx line-start (1+ (any " \t")) "File \"" @@ -1956,6 +2033,148 @@ virtualenv." :type '(alist string) :group 'python) +(defmacro python-shell--add-to-path-with-priority (pathvar paths) + "Modify PATHVAR and ensure PATHS are added only once at beginning." + `(dolist (path (reverse ,paths)) + (cl-delete path ,pathvar :test #'string=) + (cl-pushnew path ,pathvar :test #'string=))) + +(defun python-shell-calculate-pythonpath () + "Calculate the PYTHONPATH using `python-shell-extra-pythonpaths'." + (let ((pythonpath + (tramp-compat-split-string + (or (getenv "PYTHONPATH") "") path-separator))) + (python-shell--add-to-path-with-priority + pythonpath python-shell-extra-pythonpaths) + (mapconcat 'identity pythonpath path-separator))) + +(defun python-shell-calculate-process-environment () + "Calculate `process-environment' or `tramp-remote-process-environment'. +Prepends `python-shell-process-environment', sets extra +pythonpaths from `python-shell-extra-pythonpaths' and sets a few +virtualenv related vars. If `default-directory' points to a +remote host, the returned value is intended for +`tramp-remote-process-environment'." + (let* ((remote-p (file-remote-p default-directory)) + (process-environment (if remote-p + tramp-remote-process-environment + process-environment)) + (virtualenv (when python-shell-virtualenv-root + (directory-file-name python-shell-virtualenv-root)))) + (dolist (env python-shell-process-environment) + (pcase-let ((`(,key ,value) (split-string env "="))) + (setenv key value))) + (when python-shell-unbuffered + (setenv "PYTHONUNBUFFERED" "1")) + (when python-shell-extra-pythonpaths + (setenv "PYTHONPATH" (python-shell-calculate-pythonpath))) + (if (not virtualenv) + process-environment + (setenv "PYTHONHOME" nil) + (setenv "VIRTUAL_ENV" virtualenv)) + process-environment)) + +(defun python-shell-calculate-exec-path () + "Calculate `exec-path'. +Prepends `python-shell-exec-path' and adds the binary directory +for virtualenv if `python-shell-virtualenv-root' is set. If +`default-directory' points to a remote host, the returned value +appends `python-shell-remote-exec-path' instead of `exec-path'." + (let ((new-path (copy-sequence + (if (file-remote-p default-directory) + python-shell-remote-exec-path + exec-path)))) + (python-shell--add-to-path-with-priority + new-path python-shell-exec-path) + (if (not python-shell-virtualenv-root) + new-path + (python-shell--add-to-path-with-priority + new-path + (list (expand-file-name "bin" python-shell-virtualenv-root))) + new-path))) + +(defun python-shell-tramp-refresh-remote-path (vec paths) + "Update VEC's remote-path giving PATHS priority." + (let ((remote-path (tramp-get-connection-property vec "remote-path" nil))) + (when remote-path + (python-shell--add-to-path-with-priority remote-path paths) + (tramp-set-connection-property vec "remote-path" remote-path) + (tramp-set-remote-path vec)))) + +(defun python-shell-tramp-refresh-process-environment (vec env) + "Update VEC's process environment with ENV." + ;; Stolen from `tramp-open-connection-setup-interactive-shell'. + (let ((env (append (when (fboundp #'tramp-get-remote-locale) + ;; Emacs<24.4 compat. + (list (tramp-get-remote-locale vec))) + (copy-sequence env))) + (tramp-end-of-heredoc + (if (boundp 'tramp-end-of-heredoc) + tramp-end-of-heredoc + (md5 tramp-end-of-output))) + unset vars item) + (while env + (setq item (tramp-compat-split-string (car env) "=")) + (setcdr item (mapconcat 'identity (cdr item) "=")) + (if (and (stringp (cdr item)) (not (string-equal (cdr item) ""))) + (push (format "%s %s" (car item) (cdr item)) vars) + (push (car item) unset)) + (setq env (cdr env))) + (when vars + (tramp-send-command + vec + (format "while read var val; do export $var=$val; done <<'%s'\n%s\n%s" + tramp-end-of-heredoc + (mapconcat 'identity vars "\n") + tramp-end-of-heredoc) + t)) + (when unset + (tramp-send-command + vec (format "unset %s" (mapconcat 'identity unset " ")) t)))) + +(defmacro python-shell-with-environment (&rest body) + "Modify shell environment during execution of BODY. +Temporarily sets `process-environment' and `exec-path' during +execution of body. If `default-directory' points to a remote +machine then modifies `tramp-remote-process-environment' and +`python-shell-remote-exec-path' instead." + (declare (indent 0) (debug (body))) + (let ((vec (make-symbol "vec"))) + `(progn + (let* ((,vec + (when (file-remote-p default-directory) + (ignore-errors + (tramp-dissect-file-name default-directory 'noexpand)))) + (process-environment + (if ,vec + process-environment + (python-shell-calculate-process-environment))) + (exec-path + (if ,vec + exec-path + (python-shell-calculate-exec-path))) + (tramp-remote-process-environment + (if ,vec + (python-shell-calculate-process-environment) + tramp-remote-process-environment))) + (when (tramp-get-connection-process ,vec) + ;; For already existing connections, the new exec path must + ;; be re-set, otherwise it won't take effect. One example + ;; of such case is when remote dir-locals are read and + ;; *then* subprocesses are triggered within the same + ;; connection. + (python-shell-tramp-refresh-remote-path + ,vec (python-shell-calculate-exec-path)) + ;; The `tramp-remote-process-environment' variable is only + ;; effective when the started process is an interactive + ;; shell, otherwise (like in the case of processes started + ;; with `process-file') the environment is not changed. + ;; This makes environment modifications effective + ;; unconditionally. + (python-shell-tramp-refresh-process-environment + ,vec tramp-remote-process-environment)) + ,(macroexp-progn body))))) + (defvar python-shell--prompt-calculated-input-regexp nil "Calculated input prompt regexp for inferior python shell. Do not set this variable directly, instead use @@ -1978,69 +2197,70 @@ shows a warning with instructions to avoid hangs and returns nil. When `python-shell-prompt-detect-enabled' is nil avoids any detection and just returns nil." (when python-shell-prompt-detect-enabled - (let* ((process-environment (python-shell-calculate-process-environment)) - (exec-path (python-shell-calculate-exec-path)) - (code (concat - "import sys\n" - "ps = [getattr(sys, 'ps%s' % i, '') for i in range(1,4)]\n" - ;; JSON is built manually for compatibility - "ps_json = '\\n[\"%s\", \"%s\", \"%s\"]\\n' % tuple(ps)\n" - "print (ps_json)\n" - "sys.exit(0)\n")) - (output - (with-temp-buffer - ;; TODO: improve error handling by using - ;; `condition-case' and displaying the error message to - ;; the user in the no-prompts warning. - (ignore-errors - (let ((code-file (python-shell--save-temp-file code))) - ;; Use `process-file' as it is remote-host friendly. - (process-file - python-shell-interpreter - code-file - '(t nil) - nil - python-shell-interpreter-interactive-arg) - ;; Try to cleanup - (delete-file code-file))) - (buffer-string))) - (prompts - (catch 'prompts - (dolist (line (split-string output "\n" t)) - (let ((res - ;; Check if current line is a valid JSON array - (and (string= (substring line 0 2) "[\"") - (ignore-errors - ;; Return prompts as a list, not vector - (append (json-read-from-string line) nil))))) - ;; The list must contain 3 strings, where the first - ;; is the input prompt, the second is the block - ;; prompt and the last one is the output prompt. The - ;; input prompt is the only one that can't be empty. - (when (and (= (length res) 3) - (cl-every #'stringp res) - (not (string= (car res) ""))) - (throw 'prompts res)))) - nil))) - (when (and (not prompts) - python-shell-prompt-detect-failure-warning) - (lwarn - '(python python-shell-prompt-regexp) - :warning - (concat - "Python shell prompts cannot be detected.\n" - "If your emacs session hangs when starting python shells\n" - "recover with `keyboard-quit' and then try fixing the\n" - "interactive flag for your interpreter by adjusting the\n" - "`python-shell-interpreter-interactive-arg' or add regexps\n" - "matching shell prompts in the directory-local friendly vars:\n" - " + `python-shell-prompt-regexp'\n" - " + `python-shell-prompt-block-regexp'\n" - " + `python-shell-prompt-output-regexp'\n" - "Or alternatively in:\n" - " + `python-shell-prompt-input-regexps'\n" - " + `python-shell-prompt-output-regexps'"))) - prompts))) + (python-shell-with-environment + (let* ((code (concat + "import sys\n" + "ps = [getattr(sys, 'ps%s' % i, '') for i in range(1,4)]\n" + ;; JSON is built manually for compatibility + "ps_json = '\\n[\"%s\", \"%s\", \"%s\"]\\n' % tuple(ps)\n" + "print (ps_json)\n" + "sys.exit(0)\n")) + (interpreter python-shell-interpreter) + (interpreter-arg python-shell-interpreter-interactive-arg) + (output + (with-temp-buffer + ;; TODO: improve error handling by using + ;; `condition-case' and displaying the error message to + ;; the user in the no-prompts warning. + (ignore-errors + (let ((code-file (python-shell--save-temp-file code))) + ;; Use `process-file' as it is remote-host friendly. + (process-file + interpreter + code-file + '(t nil) + nil + interpreter-arg) + ;; Try to cleanup + (delete-file code-file))) + (buffer-string))) + (prompts + (catch 'prompts + (dolist (line (split-string output "\n" t)) + (let ((res + ;; Check if current line is a valid JSON array + (and (string= (substring line 0 2) "[\"") + (ignore-errors + ;; Return prompts as a list, not vector + (append (json-read-from-string line) nil))))) + ;; The list must contain 3 strings, where the first + ;; is the input prompt, the second is the block + ;; prompt and the last one is the output prompt. The + ;; input prompt is the only one that can't be empty. + (when (and (= (length res) 3) + (cl-every #'stringp res) + (not (string= (car res) ""))) + (throw 'prompts res)))) + nil))) + (when (and (not prompts) + python-shell-prompt-detect-failure-warning) + (lwarn + '(python python-shell-prompt-regexp) + :warning + (concat + "Python shell prompts cannot be detected.\n" + "If your emacs session hangs when starting python shells\n" + "recover with `keyboard-quit' and then try fixing the\n" + "interactive flag for your interpreter by adjusting the\n" + "`python-shell-interpreter-interactive-arg' or add regexps\n" + "matching shell prompts in the directory-local friendly vars:\n" + " + `python-shell-prompt-regexp'\n" + " + `python-shell-prompt-block-regexp'\n" + " + `python-shell-prompt-output-regexp'\n" + "Or alternatively in:\n" + " + `python-shell-prompt-input-regexps'\n" + " + `python-shell-prompt-output-regexps'"))) + prompts)))) (defun python-shell-prompt-validate-regexps () "Validate all user provided regexps for prompts. @@ -2136,62 +2356,14 @@ the `buffer-name'." (defun python-shell-calculate-command () "Calculate the string used to execute the inferior Python process." - (let ((exec-path (python-shell-calculate-exec-path))) - ;; `exec-path' gets tweaked so that virtualenv's specific - ;; `python-shell-interpreter' absolute path can be found by - ;; `executable-find'. - (format "%s %s" - ;; FIXME: Why executable-find? - (shell-quote-argument - (executable-find python-shell-interpreter)) - python-shell-interpreter-args))) + (format "%s %s" + (shell-quote-argument python-shell-interpreter) + python-shell-interpreter-args)) (define-obsolete-function-alias 'python-shell-parse-command #'python-shell-calculate-command "25.1") -(defun python-shell-calculate-pythonpath () - "Calculate the PYTHONPATH using `python-shell-extra-pythonpaths'." - (let ((pythonpath (getenv "PYTHONPATH")) - (extra (mapconcat 'identity - python-shell-extra-pythonpaths - path-separator))) - (if pythonpath - (concat extra path-separator pythonpath) - extra))) - -(defun python-shell-calculate-process-environment () - "Calculate process environment given `python-shell-virtualenv-root'." - (let ((process-environment (append - python-shell-process-environment - process-environment nil)) - (virtualenv (if python-shell-virtualenv-root - (directory-file-name python-shell-virtualenv-root) - nil))) - (when python-shell-unbuffered - (setenv "PYTHONUNBUFFERED" "1")) - (when python-shell-extra-pythonpaths - (setenv "PYTHONPATH" (python-shell-calculate-pythonpath))) - (if (not virtualenv) - process-environment - (setenv "PYTHONHOME" nil) - (setenv "PATH" (format "%s/bin%s%s" - virtualenv path-separator - (or (getenv "PATH") ""))) - (setenv "VIRTUAL_ENV" virtualenv)) - process-environment)) - -(defun python-shell-calculate-exec-path () - "Calculate exec path given `python-shell-virtualenv-root'." - (let ((path (append - ;; Use nil as the tail so that the list is a full copy, - ;; this is a paranoid safeguard for side-effects. - python-shell-exec-path exec-path nil))) - (if (not python-shell-virtualenv-root) - path - (cons (expand-file-name "bin" python-shell-virtualenv-root) - path)))) - (defvar python-shell--package-depth 10) (defun python-shell-package-enable (directory package) @@ -2235,7 +2407,8 @@ banner and the initial prompt are received separately." (while t (when (not (accept-process-output process timeout)) (throw 'found nil)) - (when (looking-back regexp) + (when (looking-back + regexp (car (python-util-comint-last-prompt))) (throw 'found t)))))) (defun python-shell-comint-end-of-output-p (output) @@ -2437,6 +2610,47 @@ With argument MSG show activation/deactivation message." (python-shell-font-lock-turn-off msg)) python-shell-font-lock-enable)) +(defvar python-shell--first-prompt-received-output-buffer nil) +(defvar python-shell--first-prompt-received nil) + +(defcustom python-shell-first-prompt-hook nil + "Hook run upon first (non-pdb) shell prompt detection. +This is the place for shell setup functions that need to wait for +output. Since the first prompt is ensured, this helps the +current process to not hang waiting for output by safeguarding +interactive actions can be performed. This is useful to safely +attach setup code for long-running processes that eventually +provide a shell." + :version "25.1" + :type 'hook + :group 'python) + +(defun python-shell-comint-watch-for-first-prompt-output-filter (output) + "Run `python-shell-first-prompt-hook' when first prompt is found in OUTPUT." + (when (not python-shell--first-prompt-received) + (set (make-local-variable 'python-shell--first-prompt-received-output-buffer) + (concat python-shell--first-prompt-received-output-buffer + (ansi-color-filter-apply output))) + (when (python-shell-comint-end-of-output-p + python-shell--first-prompt-received-output-buffer) + (if (string-match-p + (concat python-shell-prompt-pdb-regexp (rx eos)) + (or python-shell--first-prompt-received-output-buffer "")) + ;; Skip pdb prompts and reset the buffer. + (setq python-shell--first-prompt-received-output-buffer nil) + (set (make-local-variable 'python-shell--first-prompt-received) t) + (setq python-shell--first-prompt-received-output-buffer nil) + (with-current-buffer (current-buffer) + (let ((inhibit-quit nil)) + (run-hooks 'python-shell-first-prompt-hook)))))) + output) + +;; Used to hold user interactive overrides to +;; `python-shell-interpreter' and `python-shell-interpreter-args' that +;; will be made buffer-local by `inferior-python-mode': +(defvar python-shell--interpreter) +(defvar python-shell--interpreter-args) + (define-derived-mode inferior-python-mode comint-mode "Inferior Python" "Major mode for Python inferior process. Runs a Python interpreter as a subprocess of Emacs, with Python @@ -2462,15 +2676,16 @@ initialization of the interpreter via `python-shell-setup-codes' variable. \(Type \\[describe-mode] in the process buffer for a list of commands.)" - (let ((interpreter python-shell-interpreter) - (args python-shell-interpreter-args)) - (when python-shell--parent-buffer - (python-util-clone-local-variables python-shell--parent-buffer)) - ;; Users can override default values for these vars when calling - ;; `run-python'. This ensures new values let-bound in - ;; `python-shell-make-comint' are locally set. - (set (make-local-variable 'python-shell-interpreter) interpreter) - (set (make-local-variable 'python-shell-interpreter-args) args)) + (when python-shell--parent-buffer + (python-util-clone-local-variables python-shell--parent-buffer)) + ;; Users can interactively override default values for + ;; `python-shell-interpreter' and `python-shell-interpreter-args' + ;; when calling `run-python'. This ensures values let-bound in + ;; `python-shell-make-comint' are locally set if needed. + (set (make-local-variable 'python-shell-interpreter) + (or python-shell--interpreter python-shell-interpreter)) + (set (make-local-variable 'python-shell-interpreter-args) + (or python-shell--interpreter-args python-shell-interpreter-args)) (set (make-local-variable 'python-shell--prompt-calculated-input-regexp) nil) (set (make-local-variable 'python-shell--prompt-calculated-output-regexp) nil) (python-shell-prompt-set-calculated-regexps) @@ -2479,6 +2694,7 @@ variable. (setq mode-line-process '(":%s")) (set (make-local-variable 'comint-output-filter-functions) '(ansi-color-process-output + python-shell-comint-watch-for-first-prompt-output-filter python-pdbtrack-comint-output-filter-function python-comint-postoutput-scroll-to-bottom)) (set (make-local-variable 'compilation-error-regexp-alist) @@ -2492,9 +2708,7 @@ variable. (make-local-variable 'python-shell-internal-last-output) (when python-shell-font-lock-enable (python-shell-font-lock-turn-on)) - (compilation-shell-minor-mode 1) - (python-shell-accept-process-output - (get-buffer-process (current-buffer)))) + (compilation-shell-minor-mode 1)) (defun python-shell-make-comint (cmd proc-name &optional show internal) "Create a Python shell comint buffer. @@ -2508,31 +2722,30 @@ convention for temporary/internal buffers, and also makes sure the user is not queried for confirmation when the process is killed." (save-excursion - (let* ((proc-buffer-name - (format (if (not internal) "*%s*" " *%s*") proc-name)) - (process-environment (python-shell-calculate-process-environment)) - (exec-path (python-shell-calculate-exec-path))) - (when (not (comint-check-proc proc-buffer-name)) - (let* ((cmdlist (split-string-and-unquote cmd)) - (interpreter (car cmdlist)) - (args (cdr cmdlist)) - (buffer (apply #'make-comint-in-buffer proc-name proc-buffer-name - interpreter nil args)) - (python-shell--parent-buffer (current-buffer)) - (process (get-buffer-process buffer)) - ;; As the user may have overridden default values for - ;; these vars on `run-python', let-binding them allows - ;; to have the new right values in all setup code - ;; that's is done in `inferior-python-mode', which is - ;; important, especially for prompt detection. - (python-shell-interpreter interpreter) - (python-shell-interpreter-args - (mapconcat #'identity args " "))) - (with-current-buffer buffer - (inferior-python-mode)) - (when show (display-buffer buffer)) - (and internal (set-process-query-on-exit-flag process nil)))) - proc-buffer-name))) + (python-shell-with-environment + (let* ((proc-buffer-name + (format (if (not internal) "*%s*" " *%s*") proc-name))) + (when (not (comint-check-proc proc-buffer-name)) + (let* ((cmdlist (split-string-and-unquote cmd)) + (interpreter (car cmdlist)) + (args (cdr cmdlist)) + (buffer (apply #'make-comint-in-buffer proc-name proc-buffer-name + interpreter nil args)) + (python-shell--parent-buffer (current-buffer)) + (process (get-buffer-process buffer)) + ;; Users can override the interpreter and args + ;; interactively when calling `run-python', let-binding + ;; these allows having the new right values in all + ;; setup code that is done in `inferior-python-mode', + ;; which is important, especially for prompt detection. + (python-shell--interpreter interpreter) + (python-shell--interpreter-args + (mapconcat #'identity args " "))) + (with-current-buffer buffer + (inferior-python-mode)) + (when show (display-buffer buffer)) + (and internal (set-process-query-on-exit-flag process nil)))) + proc-buffer-name)))) ;;;###autoload (defun run-python (&optional cmd dedicated show) @@ -2612,7 +2825,8 @@ of `error' with a user-friendly message." (or (python-shell-get-process) (if interactivep (user-error - "Start a Python process first with `M-x run-python' or `%s'." + "Start a Python process first with `%s' or `%s'." + (substitute-command-keys "\\[run-python]") ;; Get the binding. (key-description (where-is-internal @@ -2771,36 +2985,39 @@ This is a wrapper over `buffer-substring' that takes care of different transformations for the code sent to be evaluated in the python shell: 1. When optional argument NOMAIN is non-nil everything under an - \"if __name__ == '__main__'\" block will be removed. + \"if __name__ == \\='__main__\\='\" block will be removed. 2. When a subregion of the buffer is sent, it takes care of appending extra empty lines so tracebacks are correct. 3. When the region sent is a substring of the current buffer, a coding cookie is added. 4. Wraps indented regions under an \"if True:\" block so the interpreter evaluates them correctly." - (let* ((substring (buffer-substring-no-properties start end)) + (let* ((start (save-excursion + ;; Normalize start to the line beginning position. + (goto-char start) + (line-beginning-position))) + (substring (buffer-substring-no-properties start end)) (starts-at-point-min-p (save-restriction (widen) (= (point-min) start))) (encoding (python-info-encoding)) + (toplevel-p (zerop (save-excursion + (goto-char start) + (python-util-forward-comment 1) + (current-indentation)))) (fillstr (when (not starts-at-point-min-p) (concat (format "# -*- coding: %s -*-\n" encoding) (make-string ;; Subtract 2 because of the coding cookie. - (- (line-number-at-pos start) 2) ?\n)))) - (toplevel-block-p (save-excursion - (goto-char start) - (or (zerop (line-number-at-pos start)) - (progn - (python-util-forward-comment 1) - (zerop (current-indentation))))))) + (- (line-number-at-pos start) 2) ?\n))))) (with-temp-buffer (python-mode) - (if fillstr (insert fillstr)) + (when fillstr + (insert fillstr)) (insert substring) (goto-char (point-min)) - (when (not toplevel-block-p) + (when (not toplevel-p) (insert "if True:") (delete-region (point) (line-end-position))) (when nomain @@ -2835,7 +3052,7 @@ the python shell: (defun python-shell-send-region (start end &optional send-main msg) "Send the region delimited by START and END to inferior Python process. When optional argument SEND-MAIN is non-nil, allow execution of -code inside blocks delimited by \"if __name__== '__main__':\". +code inside blocks delimited by \"if __name__== \\='__main__\\=':\". When called interactively SEND-MAIN defaults to nil, unless it's called with prefix argument. When optional argument MSG is non-nil, forces display of a user-friendly message if there's no @@ -2852,7 +3069,7 @@ process running; defaults to t when called interactively." (defun python-shell-send-buffer (&optional send-main msg) "Send the entire buffer to inferior Python process. When optional argument SEND-MAIN is non-nil, allow execution of -code inside blocks delimited by \"if __name__== '__main__':\". +code inside blocks delimited by \"if __name__== \\='__main__\\=':\". When called interactively SEND-MAIN defaults to nil, unless it's called with prefix argument. When optional argument MSG is non-nil, forces display of a user-friendly message if there's no @@ -2943,69 +3160,82 @@ t when called interactively." "Send all setup code for shell. This function takes the list of setup code to send from the `python-shell-setup-codes' list." - (let ((process (python-shell-get-process)) - (code (concat - (mapconcat - (lambda (elt) - (cond ((stringp elt) elt) - ((symbolp elt) (symbol-value elt)) - (t ""))) - python-shell-setup-codes - "\n\n") - "\n\nprint ('python.el: sent setup code')"))) - (python-shell-send-string code process) - (python-shell-accept-process-output process))) - -(add-hook 'inferior-python-mode-hook + (when python-shell-setup-codes + (let ((process (python-shell-get-process)) + (code (concat + (mapconcat + (lambda (elt) + (cond ((stringp elt) elt) + ((symbolp elt) (symbol-value elt)) + (t ""))) + python-shell-setup-codes + "\n\nprint ('python.el: sent setup code')")))) + (python-shell-send-string code process) + (python-shell-accept-process-output process)))) + +(add-hook 'python-shell-first-prompt-hook #'python-shell-send-setup-code) ;;; Shell completion (defcustom python-shell-completion-setup-code - "try: - import __builtin__ -except ImportError: - # Python 3 - import builtins as __builtin__ -try: - import readline, rlcompleter -except: - def __PYTHON_EL_get_completions(text): - return [] -else: - def __PYTHON_EL_get_completions(text): - builtins = dir(__builtin__) - completions = [] + " +def __PYTHON_EL_get_completions(text): + completions = [] + completer = None + + try: + import readline + try: - splits = text.split() - is_module = splits and splits[0] in ('from', 'import') - is_ipython = ('__IPYTHON__' in builtins or - '__IPYTHON__active' in builtins) - if is_module: - from IPython.core.completerlib import module_completion - completions = module_completion(text.strip()) - elif is_ipython and '__IP' in builtins: - completions = __IP.complete(text) - elif is_ipython and 'get_ipython' in builtins: - completions = get_ipython().Completer.all_completions(text) - else: - i = 0 - while True: - res = readline.get_completer()(text, i) - if not res: - break - i += 1 - completions.append(res) - except: - pass - return completions" + import __builtin__ + except ImportError: + # Python 3 + import builtins as __builtin__ + builtins = dir(__builtin__) + + is_ipython = ('__IPYTHON__' in builtins or + '__IPYTHON__active' in builtins) + splits = text.split() + is_module = splits and splits[0] in ('from', 'import') + + if is_ipython and is_module: + from IPython.core.completerlib import module_completion + completions = module_completion(text.strip()) + elif is_ipython and '__IP' in builtins: + completions = __IP.complete(text) + elif is_ipython and 'get_ipython' in builtins: + completions = get_ipython().Completer.all_completions(text) + else: + # Try to reuse current completer. + completer = readline.get_completer() + if not completer: + # importing rlcompleter sets the completer, use it as a + # last resort to avoid breaking customizations. + import rlcompleter + completer = readline.get_completer() + if getattr(completer, 'PYTHON_EL_WRAPPED', False): + completer.print_mode = False + i = 0 + while True: + completion = completer(text, i) + if not completion: + break + i += 1 + completions.append(completion) + except: + pass + finally: + if getattr(completer, 'PYTHON_EL_WRAPPED', False): + completer.print_mode = True + return completions" "Code used to setup completion in inferior Python processes." :type 'string :group 'python) (defcustom python-shell-completion-string-code - "';'.join(__PYTHON_EL_get_completions('''%s'''))\n" + "';'.join(__PYTHON_EL_get_completions('''%s'''))" "Python code used to get a string of completions separated by semicolons. The string passed to the function is the current python name or the full statement in the case of imports." @@ -3029,14 +3259,22 @@ the full statement in the case of imports." (list "pypy") "List of disabled interpreters. When a match is found, native completion is disabled." + :version "25.1" :type '(repeat string)) (defcustom python-shell-completion-native-enable t "Enable readline based native completion." + :version "25.1" :type 'boolean) -(defcustom python-shell-completion-native-output-timeout 0.01 +(defcustom python-shell-completion-native-output-timeout 5.0 "Time in seconds to wait for completion output before giving up." + :version "25.1" + :type 'float) + +(defcustom python-shell-completion-native-try-output-timeout 1.0 + "Time in seconds to wait for *trying* native completion output." + :version "25.1" :type 'float) (defvar python-shell-completion-native-redirect-buffer @@ -3052,41 +3290,131 @@ When a match is found, native completion is disabled." (defun python-shell-completion-native-try () "Return non-nil if can trigger native completion." - (let ((python-shell-completion-native-enable t)) + (let ((python-shell-completion-native-enable t) + (python-shell-completion-native-output-timeout + python-shell-completion-native-try-output-timeout)) (python-shell-completion-native-get-completions (get-buffer-process (current-buffer)) - nil "int"))) + nil ""))) (defun python-shell-completion-native-setup () "Try to setup native completion, return non-nil on success." (let ((process (python-shell-get-process))) - (python-shell-send-string - (funcall - 'mapconcat - #'identity - (list - "try:" - " import readline, rlcompleter" - ;; Remove parens on callables as it breaks completion on - ;; arguments (e.g. str(Ari)). - " class Completer(rlcompleter.Completer):" - " def _callable_postfix(self, val, word):" - " return word" - " readline.set_completer(Completer().complete)" - " if readline.__doc__ and 'libedit' in readline.__doc__:" - " readline.parse_and_bind('bind ^I rl_complete')" - " else:" - " readline.parse_and_bind('tab: complete')" - " print ('python.el: readline is available')" - "except:" - " print ('python.el: readline not available')") - "\n") - process) - (python-shell-accept-process-output process) - (when (save-excursion - (re-search-backward - (regexp-quote "python.el: readline is available") nil t 1)) - (python-shell-completion-native-try)))) + (with-current-buffer (process-buffer process) + (python-shell-send-string " +def __PYTHON_EL_native_completion_setup(): + try: + import readline + + try: + import __builtin__ + except ImportError: + # Python 3 + import builtins as __builtin__ + + builtins = dir(__builtin__) + is_ipython = ('__IPYTHON__' in builtins or + '__IPYTHON__active' in builtins) + + class __PYTHON_EL_Completer: + '''Completer wrapper that prints candidates to stdout. + + It wraps an existing completer function and changes its behavior so + that the user input is unchanged and real candidates are printed to + stdout. + + Returned candidates are '0__dummy_completion__' and + '1__dummy_completion__' in that order ('0__dummy_completion__' is + returned repeatedly until all possible candidates are consumed). + + The real candidates are printed to stdout so that they can be + easily retrieved through comint output redirect trickery. + ''' + + PYTHON_EL_WRAPPED = True + + def __init__(self, completer): + self.completer = completer + self.last_completion = None + self.print_mode = True + + def __call__(self, text, state): + if state == 0: + # Set the first dummy completion. + self.last_completion = None + completion = '0__dummy_completion__' + else: + completion = self.completer(text, state - 1) + + if not completion: + if self.last_completion != '1__dummy_completion__': + # When no more completions are available, returning a + # dummy with non-sharing prefix allow ensuring output + # while preventing changes to current input. + # Coincidentally it's also the end of output. + completion = '1__dummy_completion__' + elif completion.endswith('('): + # Remove parens on callables as it breaks completion on + # arguments (e.g. str(Ari)). + completion = completion[:-1] + self.last_completion = completion + + if completion in ( + '0__dummy_completion__', '1__dummy_completion__'): + return completion + elif completion: + # For every non-dummy completion, return a repeated dummy + # one and print the real candidate so it can be retrieved + # by comint output filters. + if self.print_mode: + print (completion) + return '0__dummy_completion__' + else: + return completion + else: + return completion + + completer = readline.get_completer() + + if not completer: + # Used as last resort to avoid breaking customizations. + import rlcompleter + completer = readline.get_completer() + + if completer and not getattr(completer, 'PYTHON_EL_WRAPPED', False): + # Wrap the existing completer function only once. + new_completer = __PYTHON_EL_Completer(completer) + if not is_ipython: + readline.set_completer(new_completer) + else: + # Try both initializations to cope with all IPython versions. + # This works fine for IPython 3.x but not for earlier: + readline.set_completer(new_completer) + # IPython<3 hacks readline such that `readline.set_completer` + # won't work. This workaround injects the new completer + # function into the existing instance directly: + instance = getattr(completer, 'im_self', completer.__self__) + instance.rlcomplete = new_completer + + if readline.__doc__ and 'libedit' in readline.__doc__: + readline.parse_and_bind('bind ^I rl_complete') + else: + readline.parse_and_bind('tab: complete') + # Require just one tab to send output. + readline.parse_and_bind('set show-all-if-ambiguous on') + + print ('python.el: native completion setup loaded') + except: + print ('python.el: native completion setup failed') + +__PYTHON_EL_native_completion_setup()" process) + (when (and + (python-shell-accept-process-output + process python-shell-completion-native-try-output-timeout) + (save-excursion + (re-search-backward + (regexp-quote "python.el: native completion setup loaded") nil t 1))) + (python-shell-completion-native-try))))) (defun python-shell-completion-native-turn-off (&optional msg) "Turn off shell native completions. @@ -3123,7 +3451,7 @@ With argument MSG show activation/deactivation message." (concat "Your `python-shell-interpreter' doesn't seem to " "support readline, yet `python-shell-completion-native' " - (format "was `t' and %S is not part of the " + (format "was t and %S is not part of the " (file-name-nondirectory python-shell-interpreter)) "`python-shell-completion-native-disabled-interpreters' " "list. Native completions have been disabled locally. ")) @@ -3133,7 +3461,7 @@ With argument MSG show activation/deactivation message." "Like `python-shell-completion-native-turn-on-maybe' but force messages." (python-shell-completion-native-turn-on-maybe t)) -(add-hook 'inferior-python-mode-hook +(add-hook 'python-shell-first-prompt-hook #'python-shell-completion-native-turn-on-maybe-with-msg) (defun python-shell-completion-native-toggle (&optional msg) @@ -3151,103 +3479,75 @@ With argument MSG show activation/deactivation message." When IMPORT is non-nil takes precedence over INPUT for completion." (with-current-buffer (process-buffer process) - (when (and python-shell-completion-native-enable - (python-util-comint-last-prompt) - (>= (point) (cdr (python-util-comint-last-prompt)))) - (let* ((input (or import input)) - (original-filter-fn (process-filter process)) - (redirect-buffer (get-buffer-create - python-shell-completion-native-redirect-buffer)) - (separators (python-rx - (or whitespace open-paren close-paren))) - (trigger "\t\t\t") - (new-input (concat input trigger)) - (input-length - (save-excursion - (+ (- (point-max) (comint-bol)) (length new-input)))) - (delete-line-command (make-string input-length ?\b)) - (input-to-send (concat new-input delete-line-command))) - ;; Ensure restoring the process filter, even if the user quits - ;; or there's some other error. - (unwind-protect - (with-current-buffer redirect-buffer - ;; Cleanup the redirect buffer - (delete-region (point-min) (point-max)) - ;; Mimic `comint-redirect-send-command', unfortunately it - ;; can't be used here because it expects a newline in the - ;; command and that's exactly what we are trying to avoid. - (let ((comint-redirect-echo-input nil) - (comint-redirect-verbose nil) - (comint-redirect-perform-sanity-check nil) - (comint-redirect-insert-matching-regexp nil) - ;; Feed it some regex that will never match. - (comint-redirect-finished-regexp "^\\'$") - (comint-redirect-output-buffer redirect-buffer)) - ;; Compatibility with Emacs 24.x. Comint changed and - ;; now `comint-redirect-filter' gets 3 args. This - ;; checks which version of `comint-redirect-filter' is - ;; in use based on its args and uses `apply-partially' - ;; to make it up for the 3 args case. - (if (= (length - (help-function-arglist 'comint-redirect-filter)) 3) - (set-process-filter - process (apply-partially - #'comint-redirect-filter original-filter-fn)) - (set-process-filter process #'comint-redirect-filter)) - (process-send-string process input-to-send) - (accept-process-output - process - python-shell-completion-native-output-timeout) - ;; XXX: can't use `python-shell-accept-process-output' - ;; here because there are no guarantees on how output - ;; ends. The workaround here is to call - ;; `accept-process-output' until we don't find anything - ;; else to accept. - (while (accept-process-output - process - python-shell-completion-native-output-timeout)) + (let* ((input (or import input)) + (original-filter-fn (process-filter process)) + (redirect-buffer (get-buffer-create + python-shell-completion-native-redirect-buffer)) + (trigger "\t") + (new-input (concat input trigger)) + (input-length + (save-excursion + (+ (- (point-max) (comint-bol)) (length new-input)))) + (delete-line-command (make-string input-length ?\b)) + (input-to-send (concat new-input delete-line-command))) + ;; Ensure restoring the process filter, even if the user quits + ;; or there's some other error. + (unwind-protect + (with-current-buffer redirect-buffer + ;; Cleanup the redirect buffer + (erase-buffer) + ;; Mimic `comint-redirect-send-command', unfortunately it + ;; can't be used here because it expects a newline in the + ;; command and that's exactly what we are trying to avoid. + (let ((comint-redirect-echo-input nil) + (comint-redirect-completed nil) + (comint-redirect-perform-sanity-check nil) + (comint-redirect-insert-matching-regexp t) + (comint-redirect-finished-regexp + "1__dummy_completion__[[:space:]]*\n") + (comint-redirect-output-buffer redirect-buffer)) + ;; Compatibility with Emacs 24.x. Comint changed and + ;; now `comint-redirect-filter' gets 3 args. This + ;; checks which version of `comint-redirect-filter' is + ;; in use based on its args and uses `apply-partially' + ;; to make it up for the 3 args case. + (if (= (length + (help-function-arglist 'comint-redirect-filter)) 3) + (set-process-filter + process (apply-partially + #'comint-redirect-filter original-filter-fn)) + (set-process-filter process #'comint-redirect-filter)) + (process-send-string process input-to-send) + ;; Grab output until our dummy completion used as + ;; output end marker is found. + (when (python-shell-accept-process-output + process python-shell-completion-native-output-timeout + comint-redirect-finished-regexp) + (re-search-backward "0__dummy_completion__" nil t) (cl-remove-duplicates (split-string (buffer-substring-no-properties - (point-min) (point-max)) - separators t)))) - (set-process-filter process original-filter-fn)))))) + (line-beginning-position) (point-min)) + "[ \f\t\n\r\v()]+" t) + :test #'string=)))) + (set-process-filter process original-filter-fn))))) (defun python-shell-completion-get-completions (process import input) "Do completion at point using PROCESS for IMPORT or INPUT. When IMPORT is non-nil takes precedence over INPUT for completion." + (setq input (or import input)) (with-current-buffer (process-buffer process) - (let* ((prompt - (let ((prompt-boundaries (python-util-comint-last-prompt))) - (buffer-substring-no-properties - (car prompt-boundaries) (cdr prompt-boundaries)))) - (completion-code - ;; Check whether a prompt matches a pdb string, an import - ;; statement or just the standard prompt and use the - ;; correct python-shell-completion-*-code string - (cond ((and (string-match - (concat "^" python-shell-prompt-pdb-regexp) prompt)) - ;; Since there are no guarantees the user will remain - ;; in the same context where completion code was sent - ;; (e.g. user steps into a function), safeguard - ;; resending completion setup continuously. - (concat python-shell-completion-setup-code - "\nprint (" python-shell-completion-string-code ")")) - ((string-match - python-shell--prompt-calculated-input-regexp prompt) - python-shell-completion-string-code) - (t nil))) - (subject (or import input))) - (and completion-code - (> (length input) 0) - (let ((completions - (python-util-strip-string - (python-shell-send-string-no-output - (format completion-code subject) process)))) - (and (> (length completions) 2) - (split-string completions - "^'\\|^\"\\|;\\|'$\\|\"$" t))))))) + (let ((completions + (python-util-strip-string + (python-shell-send-string-no-output + (format + (concat python-shell-completion-setup-code + "\nprint (" python-shell-completion-string-code ")") + input) process)))) + (when (> (length completions) 2) + (split-string completions + "^'\\|^\"\\|;\\|'$\\|\"$" t))))) (defun python-shell-completion-at-point (&optional process) "Function for `completion-at-point-functions' in `inferior-python-mode'. @@ -3274,10 +3574,28 @@ using that one instead of current buffer's process." (forward-char (length (match-string-no-properties 0))) (point)))) (end (point)) + (prompt-boundaries + (with-current-buffer (process-buffer process) + (python-util-comint-last-prompt))) + (prompt + (with-current-buffer (process-buffer process) + (when prompt-boundaries + (buffer-substring-no-properties + (car prompt-boundaries) (cdr prompt-boundaries))))) (completion-fn - (if python-shell-completion-native-enable - #'python-shell-completion-native-get-completions - #'python-shell-completion-get-completions))) + (with-current-buffer (process-buffer process) + (cond ((or (null prompt) + (< (point) (cdr prompt-boundaries))) + #'ignore) + ((or (not python-shell-completion-native-enable) + ;; Even if native completion is enabled, for + ;; pdb interaction always use the fallback + ;; mechanism since the completer is changed. + ;; Also, since pdb interaction is single-line + ;; based, this is enough. + (string-match-p python-shell-prompt-pdb-regexp prompt)) + #'python-shell-completion-get-completions) + (t #'python-shell-completion-native-get-completions))))) (list start end (completion-table-dynamic (apply-partially @@ -3329,12 +3647,18 @@ Never set this variable directly, use "Set the buffer for FILE-NAME as the tracked buffer. Internally it uses the `python-pdbtrack-tracked-buffer' variable. Returns the tracked buffer." - (let ((file-buffer (get-file-buffer - (concat (file-remote-p default-directory) - file-name)))) + (let* ((file-name-prospect (concat (file-remote-p default-directory) + file-name)) + (file-buffer (get-file-buffer file-name-prospect))) (if file-buffer (setq python-pdbtrack-tracked-buffer file-buffer) - (setq file-buffer (find-file-noselect file-name)) + (cond + ((file-exists-p file-name-prospect) + (setq file-buffer (find-file-noselect file-name-prospect))) + ((and (not (equal file-name file-name-prospect)) + (file-exists-p file-name)) + ;; Fallback to a locally available copy of the file. + (setq file-buffer (find-file-noselect file-name-prospect)))) (when (not (member file-buffer python-pdbtrack-buffers-to-kill)) (add-to-list 'python-pdbtrack-buffers-to-kill file-buffer))) file-buffer)) @@ -3580,17 +3904,12 @@ JUSTIFY should be used (if applicable) as in `fill-paragraph'." (`pep-257 (and multi-line-p (cons nil 2))) (`pep-257-nn (and multi-line-p (cons nil 1))) (`symmetric (and multi-line-p (cons 1 1))))) - (docstring-p (save-excursion - ;; Consider docstrings those strings which - ;; start on a line by themselves. - (python-nav-beginning-of-statement) - (and (= (point) str-start-pos)))) (fill-paragraph-function)) (save-restriction (narrow-to-region str-start-pos str-end-pos) (fill-paragraph justify)) (save-excursion - (when (and docstring-p python-fill-docstring-style) + (when (and (python-info-docstring-p) python-fill-docstring-style) ;; Add the number of newlines indicated by the selected style ;; at the start of the docstring. (goto-char (+ str-start-pos num-quotes)) @@ -3704,8 +4023,8 @@ The skeleton will be bound to python-skeleton-NAME." (declare (indent 2)) (let* ((name (symbol-name name)) (function-name (intern (concat "python-skeleton--" name))) - (msg (format - "Add '%s' clause? " name))) + (msg (format-message + "Add `%s' clause? " name))) (when (not skel) (setq skel `(< ,(format "%s:" name) \n \n @@ -3796,13 +4115,22 @@ The skeleton will be bound to python-skeleton-NAME." ;;; FFAP (defcustom python-ffap-setup-code - "def __FFAP_get_module_path(module): + " +def __FFAP_get_module_path(objstr): try: - import os - path = __import__(module).__file__ - if path[-4:] == '.pyc' and os.path.exists(path[0:-1]): - path = path[:-1] - return path + import inspect + import os.path + # NameError exceptions are delayed until this point. + obj = eval(objstr) + module = inspect.getmodule(obj) + filename = module.__file__ + ext = os.path.splitext(filename)[1] + if ext in ('.pyc', '.pyo'): + # Point to the source file. + filename = filename[:-1] + if os.path.exists(filename): + return filename + return '' except: return ''" "Python code to get a module path." @@ -3810,7 +4138,7 @@ The skeleton will be bound to python-skeleton-NAME." :group 'python) (defcustom python-ffap-string-code - "__FFAP_get_module_path('''%s''')\n" + "__FFAP_get_module_path('''%s''')" "Python code used to get a string with the path of a module." :type 'string :group 'python) @@ -3825,9 +4153,12 @@ The skeleton will be bound to python-skeleton-NAME." nil (let ((module-file (python-shell-send-string-no-output - (format python-ffap-string-code module) process))) - (when module-file - (substring-no-properties module-file 1 -1)))))) + (concat + python-ffap-setup-code + "\nprint (" (format python-ffap-string-code module) ")") + process))) + (unless (zerop (length module-file)) + (python-util-strip-string module-file)))))) (defvar ffap-alist) @@ -3874,8 +4205,7 @@ See `python-check-command' for the default." ""))))))) (setq python-check-custom-command command) (save-some-buffers (not compilation-ask-about-save) nil) - (let ((process-environment (python-shell-calculate-process-environment)) - (exec-path (python-shell-calculate-exec-path))) + (python-shell-with-environment (compilation-start command nil (lambda (_modename) (format python-check-buffer-name command))))) @@ -3914,13 +4244,13 @@ See `python-check-command' for the default." doc = doc.splitlines()[0] except: doc = '' - print (doc)" + return doc" "Python code to setup documentation retrieval." :type 'string :group 'python) (defcustom python-eldoc-string-code - "__PYDOC_get_help('''%s''')\n" + "__PYDOC_get_help('''%s''')" "Python code used to get a string with the documentation of an object." :type 'string :group 'python) @@ -3946,15 +4276,20 @@ returns will be used. If not FORCE-PROCESS is passed what `python-shell-get-process' returns is used." (let ((process (or force-process (python-shell-get-process)))) (when process - (let ((input (or force-input - (python-eldoc--get-symbol-at-point)))) - (and input - ;; Prevent resizing the echo area when iPython is - ;; enabled. Bug#18794. - (python-util-strip-string - (python-shell-send-string-no-output - (format python-eldoc-string-code input) - process))))))) + (let* ((input (or force-input + (python-eldoc--get-symbol-at-point))) + (docstring + (when input + ;; Prevent resizing the echo area when iPython is + ;; enabled. Bug#18794. + (python-util-strip-string + (python-shell-send-string-no-output + (concat + python-eldoc-setup-code + "\nprint(" (format python-eldoc-string-code input) ")") + process))))) + (unless (zerop (length docstring)) + docstring))))) (defun python-eldoc-function () "`eldoc-documentation-function' for Python. @@ -4416,23 +4751,40 @@ where the continued line ends." (when (looking-at (python-rx block-start)) (point-marker))))) +(defun python-info-assignment-statement-p (&optional current-line-only) + "Check if current line is an assignment. +With argument CURRENT-LINE-ONLY is non-nil, don't follow any +continuations, just check the if current line is an assignment." + (save-excursion + (let ((found nil)) + (if current-line-only + (back-to-indentation) + (python-nav-beginning-of-statement)) + (while (and + (re-search-forward (python-rx not-simple-operator + assignment-operator + (group not-simple-operator)) + (line-end-position) t) + (not found)) + (save-excursion + ;; The assignment operator should not be inside a string. + (backward-char (length (match-string-no-properties 1))) + (setq found (not (python-syntax-context-type))))) + (when found + (skip-syntax-forward " ") + (point-marker))))) + +;; TODO: rename to clarify this is only for the first continuation +;; line or remove it and move its body to `python-indent-context'. (defun python-info-assignment-continuation-line-p () - "Check if current line is a continuation of an assignment. + "Check if current line is the first continuation of an assignment. When current line is continuation of another with an assignment return the point of the first non-blank character after the operator." (save-excursion (when (python-info-continuation-line-p) (forward-line -1) - (back-to-indentation) - (when (and (not (looking-at (python-rx block-start))) - (and (re-search-forward (python-rx not-simple-operator - assignment-operator - not-simple-operator) - (line-end-position) t) - (not (python-syntax-context-type)))) - (skip-syntax-forward "\s") - (point-marker))))) + (python-info-assignment-statement-p t)))) (defun python-info-looking-at-beginning-of-defun (&optional syntax-ppss) "Check if point is at `beginning-of-defun' using SYNTAX-PPSS." @@ -4457,6 +4809,46 @@ operator." (* whitespace) line-end)) (string-equal "" (match-string-no-properties 1)))) +(defun python-info-docstring-p (&optional syntax-ppss) + "Return non-nil if point is in a docstring. +When optional argument SYNTAX-PPSS is given, use that instead of +point's current `syntax-ppss'." + ;;; https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring + (save-excursion + (when (and syntax-ppss (python-syntax-context 'string syntax-ppss)) + (goto-char (nth 8 syntax-ppss))) + (python-nav-beginning-of-statement) + (let ((counter 1) + (indentation (current-indentation)) + (backward-sexp-point) + (re (concat "[uU]?[rR]?" + (python-rx string-delimiter)))) + (when (and + (not (python-info-assignment-statement-p)) + (looking-at-p re) + ;; Allow up to two consecutive docstrings only. + (>= + 2 + (progn + (while (save-excursion + (python-nav-backward-sexp) + (setq backward-sexp-point (point)) + (and (= indentation (current-indentation)) + (not (bobp)) ; Prevent infloop. + (looking-at-p + (concat "[uU]?[rR]?" + (python-rx string-delimiter))))) + ;; Previous sexp was a string, restore point. + (goto-char backward-sexp-point) + (cl-incf counter)) + counter))) + (python-util-forward-comment -1) + (python-nav-beginning-of-statement) + (cond ((bobp)) + ((python-info-assignment-statement-p) t) + ((python-info-looking-at-beginning-of-defun)) + (t nil)))))) + (defun python-info-encoding-from-cookie () "Detect current buffer's encoding from its coding cookie. Returns the encoding as a symbol." @@ -4724,6 +5116,9 @@ returned as is." "`outline-level' function for Python mode." (1+ (/ (current-indentation) python-indent-offset)))) + (set (make-local-variable 'prettify-symbols-alist) + python--prettify-symbols-alist) + (python-skeleton-add-menu-items) (make-local-variable 'python-shell-internal-buffer) @@ -4735,7 +5130,6 @@ returned as is." (provide 'python) ;; Local Variables: -;; coding: utf-8 ;; indent-tabs-mode: nil ;; End: