Merge from emacs-24; up to 2014-07-26T12:14:42Z!schwab@linux-m68k.org

[gnu-emacs] / lisp / progmodes / python.el

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index f0f67d01845fdca26fe69ed8ce410c1fb989b993..b230992fc98fcf5e061b28aa39910433e4564756 100644 (file)
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -1,11 +1,11 @@
-;;; python.el --- Python's flying circus support for Emacs
+;;; python.el --- Python's flying circus support for Emacs -*- lexical-binding: t -*-

 
 

-;; Copyright (C) 2003-2013 Free Software Foundation, Inc.
+;; Copyright (C) 2003-2014 Free Software Foundation, Inc.

 
 ;; Author: Fabián E. Gallina <fabian@anue.biz>
 ;; URL: https://github.com/fgallina/python.el
 
 ;; Author: Fabián E. Gallina <fabian@anue.biz>
 ;; URL: https://github.com/fgallina/python.el

-;; Version: 0.24.2
-;; Maintainer: FSF
+;; Version: 0.24.4
+;; Maintainer: emacs-devel@gnu.org

 ;; Created: Jul 2010
 ;; Keywords: languages
 
 ;; Created: Jul 2010
 ;; Keywords: languages
 


@@ -31,18 +31,18 @@
 ;; found in GNU/Emacs.
 
 ;; Implements Syntax highlighting, Indentation, Movement, Shell
 ;; found in GNU/Emacs.
 
 ;; Implements Syntax highlighting, Indentation, Movement, Shell

-;; interaction, Shell completion, Shell virtualenv support, Pdb
-;; tracking, Symbol completion, Skeletons, FFAP, Code Check, Eldoc,
-;; Imenu.
+;; interaction, Shell completion, Shell virtualenv support, Shell
+;; package support, Shell syntax highlighting, Pdb tracking, Symbol
+;; completion, Skeletons, FFAP, Code Check, Eldoc, Imenu.

 
 ;; Syntax highlighting: Fontification of code is provided and supports
 ;; python's triple quoted strings properly.
 
 ;; Indentation: Automatic indentation with indentation cycling is
 ;; provided, it allows you to navigate different available levels of
 
 ;; Syntax highlighting: Fontification of code is provided and supports
 ;; python's triple quoted strings properly.
 
 ;; Indentation: Automatic indentation with indentation cycling is
 ;; provided, it allows you to navigate different available levels of

-;; indentation by hitting <tab> several times.  Also when inserting a
-;; colon the `python-indent-electric-colon' command is invoked and
-;; causes the current line to be dedented automatically if needed.
+;; indentation by hitting <tab> several times.  Also electric-indent-mode
+;; is supported such that when inserting a colon the current line is
+;; dedented automatically if needed.

 
 ;; Movement: `beginning-of-defun' and `end-of-defun' functions are
 ;; properly implemented.  There are also specialized
 
 ;; Movement: `beginning-of-defun' and `end-of-defun' functions are
 ;; properly implemented.  There are also specialized


@@ -52,67 +52,90 @@
 ;; Extra functions `python-nav-forward-statement',
 ;; `python-nav-backward-statement',
 ;; `python-nav-beginning-of-statement', `python-nav-end-of-statement',
 ;; Extra functions `python-nav-forward-statement',
 ;; `python-nav-backward-statement',
 ;; `python-nav-beginning-of-statement', `python-nav-end-of-statement',

-;; `python-nav-beginning-of-block' and `python-nav-end-of-block' are
-;; included but no bound to any key.  At last but not least the
-;; specialized `python-nav-forward-sexp' allows easy navigation
-;; between code blocks.  If you prefer `cc-mode'-like `forward-sexp'
-;; movement, setting `forward-sexp-function' to nil is enough, You can
-;; do that using the `python-mode-hook':
+;; `python-nav-beginning-of-block', `python-nav-end-of-block' and
+;; `python-nav-if-name-main' are included but no bound to any key.  At
+;; last but not least the specialized `python-nav-forward-sexp' allows
+;; easy navigation between code blocks.  If you prefer `cc-mode'-like
+;; `forward-sexp' movement, setting `forward-sexp-function' to nil is
+;; enough, You can do that using the `python-mode-hook':

 
 ;; (add-hook 'python-mode-hook
 ;;           (lambda () (setq forward-sexp-function nil)))
 
 
 ;; (add-hook 'python-mode-hook
 ;;           (lambda () (setq forward-sexp-function nil)))
 

-;; Shell interaction: is provided and allows you to execute easily any
-;; block of code of your current buffer in an inferior Python process.
+;; Shell interaction: is provided and allows opening Python shells
+;; inside Emacs and executing any block of code of your current buffer
+;; in that inferior Python process.
+
+;; Besides that only the standard CPython (2.x and 3.x) shell and
+;; IPython are officially supported out of the box, the interaction
+;; should support any other readline based Python shells as well
+;; (e.g. Jython and Pypy have been reported to work).  You can change
+;; your default interpreter and commandline arguments by setting the
+;; `python-shell-interpreter' and `python-shell-interpreter-args'
+;; variables.  This example enables IPython globally:
+
+;; (setq python-shell-interpreter "ipython"
+;;       python-shell-interpreter-args "-i")
+
+;; Using the "console" subcommand to start IPython in server-client
+;; mode is known to fail intermittently due a bug on IPython itself
+;; (see URL `http://debbugs.gnu.org/cgi/bugreport.cgi?bug=18052#27').
+;; There seems to be a race condition in the IPython server (A.K.A
+;; kernel) when code is sent while it is still initializing, sometimes
+;; causing the shell to get stalled.  With that said, if an IPython
+;; kernel is already running, "console --existing" seems to work fine.
+
+;; Running IPython on Windows needs more tweaking.  The way you should
+;; set `python-shell-interpreter' and `python-shell-interpreter-args'
+;; is as follows (of course you need to modify the paths according to
+;; your system):
+
+;; (setq python-shell-interpreter "C:\\Python27\\python.exe"
+;;       python-shell-interpreter-args
+;;       "-i C:\\Python27\\Scripts\\ipython-script.py")
+
+;; If you are experiencing missing or delayed output in your shells,
+;; that's likely caused by your Operating System's pipe buffering
+;; (e.g. this is known to happen running CPython 3.3.4 in Windows 7.
+;; See URL `http://debbugs.gnu.org/cgi/bugreport.cgi?bug=17304').  To
+;; fix this, using CPython's "-u" commandline argument or setting the
+;; "PYTHONUNBUFFERED" environment variable should help: See URL
+;; `https://docs.python.org/3/using/cmdline.html#cmdoption-u'.
+
+;; The interaction relies upon having prompts for input (e.g. ">>> "
+;; and "... " in standard Python shell) and output (e.g. "Out[1]: " in
+;; IPython) detected properly.  Failing that Emacs may hang but, in
+;; the case that happens, you can recover with \\[keyboard-quit].  To
+;; avoid this issue, a two-step prompt autodetection mechanism is
+;; provided: the first step is manual and consists of a collection of
+;; regular expressions matching common prompts for Python shells
+;; stored in `python-shell-prompt-input-regexps' and
+;; `python-shell-prompt-output-regexps', and dir-local friendly vars
+;; `python-shell-prompt-regexp', `python-shell-prompt-block-regexp',
+;; `python-shell-prompt-output-regexp' which are appended to the
+;; former automatically when a shell spawns; the second step is
+;; automatic and depends on the `python-shell-prompt-detect' helper
+;; function.  See its docstring for details on global variables that
+;; modify its behavior.

 
 ;; Shell completion: hitting tab will try to complete the current
 
 ;; Shell completion: hitting tab will try to complete the current

-;; word.  Shell completion is implemented in a manner that if you
-;; change the `python-shell-interpreter' to any other (for example
-;; IPython) it should be easy to integrate another way to calculate
-;; completions.  You just need to specify your custom
-;; `python-shell-completion-setup-code' and
-;; `python-shell-completion-string-code'.
-
-;; Here is a complete example of the settings you would use for
-;; iPython 0.11:
-
-;; (setq
-;;  python-shell-interpreter "ipython"
-;;  python-shell-interpreter-args ""
-;;  python-shell-prompt-regexp "In \\[[0-9]+\\]: "
-;;  python-shell-prompt-output-regexp "Out\\[[0-9]+\\]: "
-;;  python-shell-completion-setup-code
-;;    "from IPython.core.completerlib import module_completion"
-;;  python-shell-completion-module-string-code
-;;    "';'.join(module_completion('''%s'''))\n"
-;;  python-shell-completion-string-code
-;;    "';'.join(get_ipython().Completer.all_completions('''%s'''))\n")
-
-;; For iPython 0.10 everything would be the same except for
-;; `python-shell-completion-string-code' and
-;; `python-shell-completion-module-string-code':
-
-;; (setq python-shell-completion-string-code
-;;       "';'.join(__IP.complete('''%s'''))\n"
-;;       python-shell-completion-module-string-code "")
-
-;; Unfortunately running iPython on Windows needs some more tweaking.
-;; The way you must set `python-shell-interpreter' and
-;; `python-shell-interpreter-args' is as follows:
-
-;; (setq
-;;  python-shell-interpreter "C:\\Python27\\python.exe"
-;;  python-shell-interpreter-args
-;;  "-i C:\\Python27\\Scripts\\ipython-script.py")
-
-;; That will spawn the iPython process correctly (Of course you need
-;; to modify the paths according to your system).
-
-;; Please note that the default completion system depends on the
+;; word.  Shell completion is implemented in such way that if you
+;; change the `python-shell-interpreter' it should be possible to
+;; integrate custom logic to calculate completions.  To achieve this
+;; you just need to set `python-shell-completion-setup-code' and
+;; `python-shell-completion-string-code'.  The default provided code,
+;; enables autocompletion for both CPython and IPython (and ideally
+;; any readline based Python shell).  This code depends on the

 ;; readline module, so if you are using some Operating System that
 ;; readline module, so if you are using some Operating System that

-;; bundles Python without it (like Windows) just install the
-;; pyreadline from http://ipython.scipy.org/moin/PyReadline/Intro and
-;; you should be good to go.
+;; bundles Python without it (like Windows), installing pyreadline
+;; from URL `http://ipython.scipy.org/moin/PyReadline/Intro' should
+;; suffice.  To troubleshoot why you are not getting any completions
+;; you can try the following in your Python shell:
+
+;; >>> import readline, rlcompleter
+
+;; If you see an error, then you need to either install pyreadline or
+;; setup custom code that avoids that dependency.

 
 ;; Shell virtualenv support: The shell also contains support for
 ;; virtualenvs and other special environment modifications thanks to
 
 ;; Shell virtualenv support: The shell also contains support for
 ;; virtualenvs and other special environment modifications thanks to


@@ -147,6 +170,16 @@
 ;; introduced as simple way of adding paths to the PYTHONPATH without
 ;; affecting existing values.
 
 ;; introduced as simple way of adding paths to the PYTHONPATH without
 ;; affecting existing values.
 

+;; Shell package support: you can enable a package in the current
+;; shell so that relative imports work properly using the
+;; `python-shell-package-enable' command.
+
+;; Shell syntax highlighting: when enabled current input in shell is
+;; highlighted.  The variable `python-shell-font-lock-enable' controls
+;; activation of this feature globally when shells are started.
+;; Activation/deactivation can be also controlled on the fly via the
+;; `python-shell-font-lock-toggle' command.
+

 ;; Pdb tracking: when you execute a block of code that contains some
 ;; call to pdb (or ipdb) it will prompt the block of code and will
 ;; follow the execution of pdb marking the current line with an arrow.
 ;; Pdb tracking: when you execute a block of code that contains some
 ;; call to pdb (or ipdb) it will prompt the block of code and will
 ;; follow the execution of pdb marking the current line with an arrow.


@@ -155,15 +188,13 @@
 ;; the shell completion in background so you should run
 ;; `python-shell-send-buffer' from time to time to get better results.
 
 ;; the shell completion in background so you should run
 ;; `python-shell-send-buffer' from time to time to get better results.
 

-;; Skeletons: 6 skeletons are provided for simple inserting of class,
-;; def, for, if, try and while.  These skeletons are integrated with
-;; dabbrev.  If you have `dabbrev-mode' activated and
+;; Skeletons: skeletons are provided for simple inserting of things like class,
+;; def, for, import, if, try, and while.  These skeletons are
+;; integrated with abbrev.  If you have `abbrev-mode' activated and

 ;; `python-skeleton-autoinsert' is set to t, then whenever you type
 ;; the name of any of those defined and hit SPC, they will be
 ;; automatically expanded.  As an alternative you can use the defined
 ;; `python-skeleton-autoinsert' is set to t, then whenever you type
 ;; the name of any of those defined and hit SPC, they will be
 ;; automatically expanded.  As an alternative you can use the defined

-;; skeleton commands: `python-skeleton-class', `python-skeleton-def'
-;; `python-skeleton-for', `python-skeleton-if', `python-skeleton-try'
-;; and `python-skeleton-while'.
+;; skeleton commands: `python-skeleton-<foo>'.

 
 ;; FFAP: You can find the filename for a given module when using ffap
 ;; out of the box.  This feature needs an inferior python shell
 
 ;; FFAP: You can find the filename for a given module when using ffap
 ;; out of the box.  This feature needs an inferior python shell


@@ -177,12 +208,14 @@
 ;; might guessed you should run `python-shell-send-buffer' from time
 ;; to time to get better results too.
 
 ;; might guessed you should run `python-shell-send-buffer' from time
 ;; to time to get better results too.
 

-;; Imenu: This mode supports Imenu in its most basic form, letting it
-;; build the necessary alist via `imenu-default-create-index-function'
-;; by having set `imenu-extract-index-name-function' to
-;; `python-info-current-defun' and
-;; `imenu-prev-index-position-function' to
-;; `python-imenu-prev-index-position'.
+;; Imenu: There are two index building functions to be used as
+;; `imenu-create-index-function': `python-imenu-create-index' (the
+;; default one, builds the alist in form of a tree) and
+;; `python-imenu-create-flat-index'.  See also
+;; `python-imenu-format-item-label-function',
+;; `python-imenu-format-parent-item-label-function',
+;; `python-imenu-format-parent-item-jump-label-function' variables for
+;; changing the way labels are formatted in the tree version.

 
 ;; If you used python-mode.el you probably will miss auto-indentation
 ;; when inserting newlines.  To achieve the same behavior you have
 
 ;; If you used python-mode.el you probably will miss auto-indentation
 ;; when inserting newlines.  To achieve the same behavior you have


@@ -211,7 +244,9 @@
 ;;; Code:
 
 (require 'ansi-color)
 ;;; Code:
 
 (require 'ansi-color)

+(require 'cl-lib)

 (require 'comint)
 (require 'comint)

+(require 'json)

 
 ;; Avoid compiler warnings
 (defvar view-return-to-alist)
 
 ;; Avoid compiler warnings
 (defvar view-return-to-alist)


@@ -223,7 +258,7 @@
 ;;;###autoload
 (add-to-list 'auto-mode-alist (cons (purecopy "\\.py\\'")  'python-mode))
 ;;;###autoload
 ;;;###autoload
 (add-to-list 'auto-mode-alist (cons (purecopy "\\.py\\'")  'python-mode))
 ;;;###autoload

-(add-to-list 'interpreter-mode-alist (cons (purecopy "python") 'python-mode))
+(add-to-list 'interpreter-mode-alist (cons (purecopy "python[0-9.]*") 'python-mode))

 
 (defgroup python nil
   "Python Language's flying circus support for Emacs."
 
 (defgroup python nil
   "Python Language's flying circus support for Emacs."


@@ -246,12 +281,12 @@
     (define-key map (kbd "<backtab>") 'python-indent-dedent-line)
     (define-key map "\C-c<" 'python-indent-shift-left)
     (define-key map "\C-c>" 'python-indent-shift-right)
     (define-key map (kbd "<backtab>") 'python-indent-dedent-line)
     (define-key map "\C-c<" 'python-indent-shift-left)
     (define-key map "\C-c>" 'python-indent-shift-right)

-    (define-key map ":" 'python-indent-electric-colon)

     ;; Skeletons
     (define-key map "\C-c\C-tc" 'python-skeleton-class)
     (define-key map "\C-c\C-td" 'python-skeleton-def)
     (define-key map "\C-c\C-tf" 'python-skeleton-for)
     (define-key map "\C-c\C-ti" 'python-skeleton-if)
     ;; Skeletons
     (define-key map "\C-c\C-tc" 'python-skeleton-class)
     (define-key map "\C-c\C-td" 'python-skeleton-def)
     (define-key map "\C-c\C-tf" 'python-skeleton-for)
     (define-key map "\C-c\C-ti" 'python-skeleton-if)

+    (define-key map "\C-c\C-tm" 'python-skeleton-import)

     (define-key map "\C-c\C-tt" 'python-skeleton-try)
     (define-key map "\C-c\C-tw" 'python-skeleton-while)
     ;; Shell interaction
     (define-key map "\C-c\C-tt" 'python-skeleton-try)
     (define-key map "\C-c\C-tw" 'python-skeleton-while)
     ;; Shell interaction


@@ -321,6 +356,13 @@
                                    (or "def" "class" "if" "elif" "else" "try"
                                        "except" "finally" "for" "while" "with")
                                    symbol-end))
                                    (or "def" "class" "if" "elif" "else" "try"
                                        "except" "finally" "for" "while" "with")
                                    symbol-end))

+      (dedenter            . ,(rx symbol-start
+                                   (or "elif" "else" "except" "finally")
+                                   symbol-end))
+      (block-ender         . ,(rx symbol-start
+                                  (or
+                                   "break" "continue" "pass" "raise" "return")
+                                  symbol-end))

       (decorator            . ,(rx line-start (* space) ?@ (any letter ?_)
                                    (* (any word ?_))))
       (defun                . ,(rx symbol-start (or "def" "class") symbol-end))
       (decorator            . ,(rx line-start (* space) ?@ (any letter ?_)
                                    (* (any word ?_))))
       (defun                . ,(rx symbol-start (or "def" "class") symbol-end))


@@ -356,7 +398,7 @@
 
   (defmacro python-rx (&rest regexps)
     "Python mode specialized rx macro.
 
   (defmacro python-rx (&rest regexps)
     "Python mode specialized rx macro.

-This variant of `rx' supports common python named REGEXPS."
+This variant of `rx' supports common Python named REGEXPS."

     (let ((rx-constituents (append python-rx-constituents rx-constituents)))
       (cond ((null regexps)
              (error "No regexp"))
     (let ((rx-constituents (append python-rx-constituents rx-constituents)))
       (cond ((null regexps)
              (error "No regexp"))


@@ -368,22 +410,24 @@ This variant of `rx' supports common python named REGEXPS."
 \f
 ;;; Font-lock and syntax
 
 \f
 ;;; Font-lock and syntax
 

+(eval-when-compile
+  (defun python-syntax--context-compiler-macro (form type &optional syntax-ppss)
+    (pcase type
+      (`'comment
+       `(let ((ppss (or ,syntax-ppss (syntax-ppss))))
+          (and (nth 4 ppss) (nth 8 ppss))))
+      (`'string
+       `(let ((ppss (or ,syntax-ppss (syntax-ppss))))
+          (and (nth 3 ppss) (nth 8 ppss))))
+      (`'paren
+       `(nth 1 (or ,syntax-ppss (syntax-ppss))))
+      (_ form))))
+

 (defun python-syntax-context (type &optional syntax-ppss)
   "Return non-nil if point is on TYPE using SYNTAX-PPSS.
 TYPE can be `comment', `string' or `paren'.  It returns the start
 character address of the specified TYPE."
 (defun python-syntax-context (type &optional syntax-ppss)
   "Return non-nil if point is on TYPE using SYNTAX-PPSS.
 TYPE can be `comment', `string' or `paren'.  It returns the start
 character address of the specified TYPE."

-  (declare (compiler-macro
-            (lambda (form)
-              (pcase type
-                (`'comment
-                 `(let ((ppss (or ,syntax-ppss (syntax-ppss))))
-                    (and (nth 4 ppss) (nth 8 ppss))))
-                (`'string
-                 `(let ((ppss (or ,syntax-ppss (syntax-ppss))))
-                    (and (nth 3 ppss) (nth 8 ppss))))
-                (`'paren
-                 `(nth 1 (or ,syntax-ppss (syntax-ppss))))
-                (_ form)))))
+  (declare (compiler-macro python-syntax--context-compiler-macro))

   (let ((ppss (or syntax-ppss (syntax-ppss))))
     (pcase type
       (`comment (and (nth 4 ppss) (nth 8 ppss)))
   (let ((ppss (or syntax-ppss (syntax-ppss))))
     (pcase type
       (`comment (and (nth 4 ppss) (nth 8 ppss)))


@@ -497,29 +541,24 @@ The type returned can be `comment', `string' or `paren'."
     (,(lambda (limit)
         (let ((re (python-rx (group (+ (any word ?. ?_)))
                              (? ?\[ (+ (not (any  ?\]))) ?\]) (* space)
     (,(lambda (limit)
         (let ((re (python-rx (group (+ (any word ?. ?_)))
                              (? ?\[ (+ (not (any  ?\]))) ?\]) (* space)

-                             assignment-operator)))
-          (when (re-search-forward re limit t)
-            (while (and (python-syntax-context 'paren)
-                        (re-search-forward re limit t)))
-            (if (not (or (python-syntax-context 'paren)
-                         (equal (char-after (point-marker)) ?=)))
-                t
-              (set-match-data nil)))))
+                             assignment-operator))
+              (res nil))
+          (while (and (setq res (re-search-forward re limit t))
+                      (or (python-syntax-context 'paren)
+                          (equal (char-after (point)) ?=))))
+          res))

      (1 font-lock-variable-name-face nil nil))
     ;; support for a, b, c = (1, 2, 3)
     (,(lambda (limit)
         (let ((re (python-rx (group (+ (any word ?. ?_))) (* space)
                              (* ?, (* space) (+ (any word ?. ?_)) (* space))
                              ?, (* space) (+ (any word ?. ?_)) (* space)
      (1 font-lock-variable-name-face nil nil))
     ;; support for a, b, c = (1, 2, 3)
     (,(lambda (limit)
         (let ((re (python-rx (group (+ (any word ?. ?_))) (* space)
                              (* ?, (* space) (+ (any word ?. ?_)) (* space))
                              ?, (* space) (+ (any word ?. ?_)) (* space)

-                             assignment-operator)))
-          (when (and (re-search-forward re limit t)
-                     (goto-char (nth 3 (match-data))))
-            (while (and (python-syntax-context 'paren)
-                        (re-search-forward re limit t))
-              (goto-char (nth 3 (match-data))))
-            (if (not (python-syntax-context 'paren))
-                t
-              (set-match-data nil)))))
+                             assignment-operator))
+              (res nil))
+          (while (and (setq res (re-search-forward re limit t))
+                      (goto-char (match-end 1))
+                      (python-syntax-context 'paren)))
+          res))

      (1 font-lock-variable-name-face nil nil))))
 
 (defconst python-syntax-propertize-function
      (1 font-lock-variable-name-face nil nil))))
 
 (defconst python-syntax-propertize-function


@@ -530,7 +569,7 @@ The type returned can be `comment', `string' or `paren'."
 (defsubst python-syntax-count-quotes (quote-char &optional point limit)
   "Count number of quotes around point (max is 3).
 QUOTE-CHAR is the quote char to count.  Optional argument POINT is
 (defsubst python-syntax-count-quotes (quote-char &optional point limit)
   "Count number of quotes around point (max is 3).
 QUOTE-CHAR is the quote char to count.  Optional argument POINT is

-the point where scan starts (defaults to current point) and LIMIT
+the point where scan starts (defaults to current point), and LIMIT

 is used to limit the scan."
   (let ((i 0))
     (while (and (< i 3)
 is used to limit the scan."
   (let ((i 0))
     (while (and (< i 3)


@@ -633,11 +672,6 @@ It makes underscores and dots word constituent chars.")
 (defvar python-indent-levels '(0)
   "Levels of indentation available for `python-indent-line-function'.")
 
 (defvar python-indent-levels '(0)
   "Levels of indentation available for `python-indent-line-function'.")
 

-(defvar python-indent-dedenters '("else" "elif" "except" "finally")
-  "List of words that should be dedented.
-These make `python-indent-calculate-indentation' subtract the value of
-`python-indent-offset'.")
-

 (defun python-indent-guess-indent-offset ()
   "Guess and set `python-indent-offset' for the current buffer."
   (interactive)
 (defun python-indent-guess-indent-offset ()
   "Guess and set `python-indent-offset' for the current buffer."
   (interactive)


@@ -670,7 +704,7 @@ These make `python-indent-calculate-indentation' subtract the value of
                  (goto-char block-end)
                  (python-util-forward-comment)
                  (current-indentation))))
                  (goto-char block-end)
                  (python-util-forward-comment)
                  (current-indentation))))

-          (if indentation
+          (if (and indentation (not (zerop indentation)))

               (set (make-local-variable 'python-indent-offset) indentation)
             (message "Can't guess python-indent-offset, using defaults: %s"
                      python-indent-offset)))))))
               (set (make-local-variable 'python-indent-offset) indentation)
             (message "Can't guess python-indent-offset, using defaults: %s"
                      python-indent-offset)))))))


@@ -678,14 +712,17 @@ These make `python-indent-calculate-indentation' subtract the value of
 (defun python-indent-context ()
   "Get information on indentation context.
 Context information is returned with a cons with the form:
 (defun python-indent-context ()
   "Get information on indentation context.
 Context information is returned with a cons with the form:

-    \(STATUS . START)
+    (STATUS . START)

 
 Where status can be any of the following symbols:
 
 Where status can be any of the following symbols:

+
+ * after-comment: When current line might continue a comment block

  * inside-paren: If point in between (), {} or []
  * inside-string: If point is inside a string
  * after-backslash: Previous line ends in a backslash
  * after-beginning-of-block: Point is after beginning of block
  * after-line: Point is after normal line
  * inside-paren: If point in between (), {} or []
  * inside-string: If point is inside a string
  * after-backslash: Previous line ends in a backslash
  * after-beginning-of-block: Point is after beginning of block
  * after-line: Point is after normal line

+ * dedenter-statement: Point is on a dedenter statement.

  * no-indent: Point is at beginning of buffer or other special case
 START is the buffer position where the sexp starts."
   (save-restriction
  * no-indent: Point is at beginning of buffer or other special case
 START is the buffer position where the sexp starts."
   (save-restriction


@@ -699,6 +736,17 @@ START is the buffer position where the sexp starts."
            (goto-char (line-beginning-position))
            (bobp))
          'no-indent)
            (goto-char (line-beginning-position))
            (bobp))
          'no-indent)

+        ;; Comment continuation
+        ((save-excursion
+           (when (and
+                  (or
+                   (python-info-current-line-comment-p)
+                   (python-info-current-line-empty-p))
+                  (progn
+                    (forward-comment -1)
+                    (python-info-current-line-comment-p)))
+             (setq start (point))
+             'after-comment)))

         ;; Inside string
         ((setq start (python-syntax-context 'string ppss))
          'inside-string)
         ;; Inside string
         ((setq start (python-syntax-context 'string ppss))
          'inside-string)


@@ -729,6 +777,8 @@ START is the buffer position where the sexp starts."
                          (when (looking-at (python-rx block-start))
                            (point-marker)))))
          'after-beginning-of-block)
                          (when (looking-at (python-rx block-start))
                            (point-marker)))))
          'after-beginning-of-block)

+        ((when (setq start (python-info-dedenter-statement-p))
+           'dedenter-statement))

         ;; After normal line
         ((setq start (save-excursion
                        (back-to-indentation)
         ;; After normal line
         ((setq start (save-excursion
                        (back-to-indentation)


@@ -750,24 +800,44 @@ START is the buffer position where the sexp starts."
       (save-excursion
         (pcase context-status
           (`no-indent 0)
       (save-excursion
         (pcase context-status
           (`no-indent 0)

+          (`after-comment
+           (goto-char context-start)
+           (current-indentation))

           ;; When point is after beginning of block just add one level
           ;; of indentation relative to the context-start
           (`after-beginning-of-block
            (goto-char context-start)
            (+ (current-indentation) python-indent-offset))
           ;; When after a simple line just use previous line
           ;; When point is after beginning of block just add one level
           ;; of indentation relative to the context-start
           (`after-beginning-of-block
            (goto-char context-start)
            (+ (current-indentation) python-indent-offset))
           ;; When after a simple line just use previous line

-          ;; indentation, in the case current line starts with a
-          ;; `python-indent-dedenters' de-indent one level.
+          ;; indentation.

           (`after-line
           (`after-line

-           (-
-            (save-excursion
-              (goto-char context-start)
-              (current-indentation))
-            (if (progn
-                  (back-to-indentation)
-                  (looking-at (regexp-opt python-indent-dedenters)))
-                python-indent-offset
-              0)))
+           (let* ((pair (save-excursion
+                          (goto-char context-start)
+                          (cons
+                           (current-indentation)
+                           (python-info-beginning-of-block-p))))
+                  (context-indentation (car pair))
+                  ;; TODO: Separate block enders into its own case.
+                  (adjustment
+                   (if (save-excursion
+                         (python-util-forward-comment -1)
+                         (python-nav-beginning-of-statement)
+                         (looking-at (python-rx block-ender)))
+                       python-indent-offset
+                     0)))
+             (- context-indentation adjustment)))
+          ;; When point is on a dedenter statement, search for the
+          ;; opening block that corresponds to it and use its
+          ;; indentation.  If no opening block is found just remove
+          ;; indentation as this is an invalid python file.
+          (`dedenter-statement
+           (let ((block-start-point
+                  (python-info-dedenter-opening-block-position)))
+             (save-excursion
+               (if (not block-start-point)
+                   0
+                 (goto-char block-start-point)
+                 (current-indentation)))))

           ;; When inside of a string, do nothing. just use the current
           ;; indentation.  XXX: perhaps it would be a good idea to
           ;; invoke standard text indentation here
           ;; When inside of a string, do nothing. just use the current
           ;; indentation.  XXX: perhaps it would be a good idea to
           ;; invoke standard text indentation here


@@ -894,16 +964,25 @@ START is the buffer position where the sexp starts."
 
 (defun python-indent-calculate-levels ()
   "Calculate `python-indent-levels' and reset `python-indent-current-level'."
 
 (defun python-indent-calculate-levels ()
   "Calculate `python-indent-levels' and reset `python-indent-current-level'."

-  (let* ((indentation (python-indent-calculate-indentation))
-         (remainder (% indentation python-indent-offset))
-         (steps (/ (- indentation remainder) python-indent-offset)))
-    (setq python-indent-levels (list 0))
-    (dotimes (step steps)
-      (push (* python-indent-offset (1+ step)) python-indent-levels))
-    (when (not (eq 0 remainder))
-      (push (+ (* python-indent-offset steps) remainder) python-indent-levels))
-    (setq python-indent-levels (nreverse python-indent-levels))
-    (setq python-indent-current-level (1- (length python-indent-levels)))))
+  (if (not (python-info-dedenter-statement-p))
+      (let* ((indentation (python-indent-calculate-indentation))
+             (remainder (% indentation python-indent-offset))
+             (steps (/ (- indentation remainder) python-indent-offset)))
+        (setq python-indent-levels (list 0))
+        (dotimes (step steps)
+          (push (* python-indent-offset (1+ step)) python-indent-levels))
+        (when (not (eq 0 remainder))
+          (push (+ (* python-indent-offset steps) remainder) python-indent-levels)))
+    (setq python-indent-levels
+          (or
+           (mapcar (lambda (pos)
+                     (save-excursion
+                       (goto-char pos)
+                       (current-indentation)))
+                   (python-info-dedenter-opening-block-positions))
+           (list 0))))
+  (setq python-indent-current-level (1- (length python-indent-levels))
+        python-indent-levels (nreverse python-indent-levels)))

 
 (defun python-indent-toggle-levels ()
   "Toggle `python-indent-current-level' over `python-indent-levels'."
 
 (defun python-indent-toggle-levels ()
   "Toggle `python-indent-current-level' over `python-indent-levels'."


@@ -926,11 +1005,11 @@ variable `python-indent-current-level'.
 
 When the variable `last-command' is not equal to one of the
 symbols inside `python-indent-trigger-commands' and FORCE-TOGGLE
 
 When the variable `last-command' is not equal to one of the
 symbols inside `python-indent-trigger-commands' and FORCE-TOGGLE

-is nil it calculates possible indentation levels and saves it in
-the variable `python-indent-levels'.  Afterwards it sets the
+is nil it calculates possible indentation levels and saves them
+in the variable `python-indent-levels'.  Afterwards it sets the

 variable `python-indent-current-level' correctly so offset is
 variable `python-indent-current-level' correctly so offset is

-equal to (`nth' `python-indent-current-level'
-`python-indent-levels')"
+equal to
+   (nth python-indent-current-level python-indent-levels)"

   (or
    (and (or (and (memq this-command python-indent-trigger-commands)
                  (eq last-command this-command))
   (or
    (and (or (and (memq this-command python-indent-trigger-commands)
                  (eq last-command this-command))


@@ -952,7 +1031,7 @@ equal to (`nth' `python-indent-current-level'
       (indent-to next-indent)
       (goto-char starting-pos))
     (and follow-indentation-p (back-to-indentation)))
       (indent-to next-indent)
       (goto-char starting-pos))
     (and follow-indentation-p (back-to-indentation)))

-  (python-info-closing-block-message))
+  (python-info-dedenter-opening-block-message))

 
 (defun python-indent-line-function ()
   "`indent-line-function' for Python mode.
 
 (defun python-indent-line-function ()
   "`indent-line-function' for Python mode.


@@ -963,9 +1042,9 @@ See `python-indent-line' for details."
   "De-indent current line."
   (interactive "*")
   (when (and (not (python-syntax-comment-or-string-p))
   "De-indent current line."
   (interactive "*")
   (when (and (not (python-syntax-comment-or-string-p))

-             (<= (point-marker) (save-excursion
+             (<= (point) (save-excursion

                                   (back-to-indentation)
                                   (back-to-indentation)

-                                  (point-marker)))
+                           (point)))

              (> (current-column) 0))
     (python-indent-line t)
     t))
              (> (current-column) 0))
     (python-indent-line t)
     t))


@@ -973,14 +1052,14 @@ See `python-indent-line' for details."
 (defun python-indent-dedent-line-backspace (arg)
   "De-indent current line.
 Argument ARG is passed to `backward-delete-char-untabify' when
 (defun python-indent-dedent-line-backspace (arg)
   "De-indent current line.
 Argument ARG is passed to `backward-delete-char-untabify' when

-point is  not in between the indentation."
+point is not in between the indentation."

   (interactive "*p")
   (when (not (python-indent-dedent-line))
     (backward-delete-char-untabify arg)))
 (put 'python-indent-dedent-line-backspace 'delete-selection 'supersede)
 
 (defun python-indent-region (start end)
   (interactive "*p")
   (when (not (python-indent-dedent-line))
     (backward-delete-char-untabify arg)))
 (put 'python-indent-dedent-line-backspace 'delete-selection 'supersede)
 
 (defun python-indent-region (start end)

-  "Indent a python region automagically.
+  "Indent a Python region automagically.

 
 Called from a program, START and END specify the region to indent."
   (let ((deactivate-mark nil))
 
 Called from a program, START and END specify the region to indent."
   (let ((deactivate-mark nil))


@@ -1032,14 +1111,12 @@ any lines in the region are indented less than COUNT columns."
         (while (< (point) end)
           (if (and (< (current-indentation) count)
                    (not (looking-at "[ \t]*$")))
         (while (< (point) end)
           (if (and (< (current-indentation) count)
                    (not (looking-at "[ \t]*$")))

-              (error "Can't shift all lines enough"))
+              (user-error "Can't shift all lines enough"))

           (forward-line))
         (indent-rigidly start end (- count))))))
 
           (forward-line))
         (indent-rigidly start end (- count))))))
 

-(add-to-list 'debug-ignored-errors "^Can't shift all lines enough")
-

 (defun python-indent-shift-right (start end &optional count)
 (defun python-indent-shift-right (start end &optional count)

-  "Shift lines contained in region START END by COUNT columns to the left.
+  "Shift lines contained in region START END by COUNT columns to the right.

 COUNT defaults to `python-indent-offset'.  If region isn't
 active, the current line is shifted.  The shifted region includes
 the lines in which START and END lie."
 COUNT defaults to `python-indent-offset'.  If region isn't
 active, the current line is shifted.  The shifted region includes
 the lines in which START and END lie."


@@ -1048,48 +1125,53 @@ the lines in which START and END lie."
        (list (region-beginning) (region-end) current-prefix-arg)
      (list (line-beginning-position) (line-end-position) current-prefix-arg)))
   (let ((deactivate-mark nil))
        (list (region-beginning) (region-end) current-prefix-arg)
      (list (line-beginning-position) (line-end-position) current-prefix-arg)))
   (let ((deactivate-mark nil))

-    (if count
-        (setq count (prefix-numeric-value count))
-      (setq count python-indent-offset))
+    (setq count (if count (prefix-numeric-value count)
+                  python-indent-offset))

     (indent-rigidly start end count)))
 
     (indent-rigidly start end count)))
 

-(defun python-indent-electric-colon (arg)
-  "Insert a colon and maybe de-indent the current line.
-With numeric ARG, just insert that many colons.  With
-\\[universal-argument], just insert a single colon."
-  (interactive "*P")
-  (self-insert-command (if (not (integerp arg)) 1 arg))
-  (when (and (not arg)
-             (eolp)
-             (not (equal ?: (char-after (- (point-marker) 2))))
-             (not (python-syntax-comment-or-string-p)))
-    (let ((indentation (current-indentation))
-          (calculated-indentation (python-indent-calculate-indentation)))
-      (python-info-closing-block-message)
-      (when (> indentation calculated-indentation)
-        (save-excursion
-          (indent-line-to calculated-indentation)
-          (when (not (python-info-closing-block-message))
-            (indent-line-to indentation)))))))
-(put 'python-indent-electric-colon 'delete-selection t)
-

 (defun python-indent-post-self-insert-function ()
 (defun python-indent-post-self-insert-function ()

-  "Adjust closing paren line indentation after a char is added.
-This function is intended to be added to the
-`post-self-insert-hook.'  If a line renders a paren alone, after
-adding a char before it, the line will be re-indented
-automatically if needed."
-  (when (and (eq (char-before) last-command-event)
-             (not (bolp))
-             (memq (char-after) '(?\) ?\] ?\})))
-    (save-excursion
-      (goto-char (line-beginning-position))
-      ;; If after going to the beginning of line the point
-      ;; is still inside a paren it's ok to do the trick
-      (when (python-syntax-context 'paren)
+  "Adjust indentation after insertion of some characters.
+This function is intended to be added to `post-self-insert-hook.'
+If a line renders a paren alone, after adding a char before it,
+the line will be re-indented automatically if needed."
+  (when (and electric-indent-mode
+             (eq (char-before) last-command-event))
+    (cond
+     ;; Electric indent inside parens
+     ((and
+       (not (bolp))
+       (let ((paren-start (python-syntax-context 'paren)))
+         ;; Check that point is inside parens.
+         (when paren-start
+           (not
+            ;; Filter the case where input is happening in the same
+            ;; line where the open paren is.
+            (= (line-number-at-pos)
+               (line-number-at-pos paren-start)))))
+       ;; When content has been added before the closing paren or a
+       ;; comma has been inserted, it's ok to do the trick.
+       (or
+        (memq (char-after) '(?\) ?\] ?\}))
+        (eq (char-before) ?,)))
+      (save-excursion
+        (goto-char (line-beginning-position))

         (let ((indentation (python-indent-calculate-indentation)))
           (when (< (current-indentation) indentation)
         (let ((indentation (python-indent-calculate-indentation)))
           (when (< (current-indentation) indentation)

-            (indent-line-to indentation)))))))
+            (indent-line-to indentation)))))
+     ;; Electric colon
+     ((and (eq ?: last-command-event)
+           (memq ?: electric-indent-chars)
+           (not current-prefix-arg)
+           ;; Trigger electric colon only at end of line
+           (eolp)
+           ;; Avoid re-indenting on extra colon
+           (not (equal ?: (char-before (1- (point)))))
+           (not (python-syntax-comment-or-string-p))
+           ;; Never re-indent at beginning of defun
+           (not (save-excursion
+                  (python-nav-beginning-of-statement)
+                  (python-info-looking-at-beginning-of-defun))))
+      (python-indent-line)))))

 
 \f
 ;;; Navigation
 
 \f
 ;;; Navigation


@@ -1144,20 +1226,20 @@ With positive ARG search backwards, else search forwards."
 
 (defun python-nav-beginning-of-defun (&optional arg)
   "Move point to `beginning-of-defun'.
 
 (defun python-nav-beginning-of-defun (&optional arg)
   "Move point to `beginning-of-defun'.

-With positive ARG search backwards else search forward.  When ARG
-is nil or 0 defaults to 1.  When searching backwards nested
-defuns are handled with care depending on current point
-position.  Return non-nil if point is moved to
+With positive ARG search backwards else search forward.
+ARG nil or 0 defaults to 1.  When searching backwards,
+nested defuns are handled with care depending on current
+point position.  Return non-nil if point is moved to

 `beginning-of-defun'."
   (when (or (null arg) (= arg 0)) (setq arg 1))
   (let ((found))
 `beginning-of-defun'."
   (when (or (null arg) (= arg 0)) (setq arg 1))
   (let ((found))

-    (cond ((and (eq this-command 'mark-defun)
-                (python-info-looking-at-beginning-of-defun)))
-          (t
-           (dotimes (i (if (> arg 0) arg (- arg)))
-             (when (and (python-nav--beginning-of-defun arg)
-                        (not found))
-               (setq found t)))))
+    (while (and (not (= arg 0))
+                (let ((keep-searching-p
+                       (python-nav--beginning-of-defun arg)))
+                  (when (and keep-searching-p (null found))
+                    (setq found t))
+                  keep-searching-p))
+      (setq arg (if (> arg 0) (1- arg) (1+ arg))))

     found))
 
 (defun python-nav-end-of-defun ()
     found))
 
 (defun python-nav-end-of-defun ()


@@ -1180,18 +1262,88 @@ Returns nil if point is not in a def or class."
       ;; Ensure point moves forward.
       (and (> beg-pos (point)) (goto-char beg-pos)))))
 
       ;; Ensure point moves forward.
       (and (> beg-pos (point)) (goto-char beg-pos)))))
 

+(defun python-nav--syntactically (fn poscompfn &optional contextfn)
+  "Move point using FN avoiding places with specific context.
+FN must take no arguments.  POSCOMPFN is a two arguments function
+used to compare current and previous point after it is moved
+using FN, this is normally a less-than or greater-than
+comparison.  Optional argument CONTEXTFN defaults to
+`python-syntax-context-type' and is used for checking current
+point context, it must return a non-nil value if this point must
+be skipped."
+  (let ((contextfn (or contextfn 'python-syntax-context-type))
+        (start-pos (point-marker))
+        (prev-pos))
+    (catch 'found
+      (while t
+        (let* ((newpos
+                (and (funcall fn) (point-marker)))
+               (context (funcall contextfn)))
+          (cond ((and (not context) newpos
+                      (or (and (not prev-pos) newpos)
+                          (and prev-pos newpos
+                               (funcall poscompfn newpos prev-pos))))
+                 (throw 'found (point-marker)))
+                ((and newpos context)
+                 (setq prev-pos (point)))
+                (t (when (not newpos) (goto-char start-pos))
+                   (throw 'found nil))))))))
+
+(defun python-nav--forward-defun (arg)
+  "Internal implementation of python-nav-{backward,forward}-defun.
+Uses ARG to define which function to call, and how many times
+repeat it."
+  (let ((found))
+    (while (and (> arg 0)
+                (setq found
+                      (python-nav--syntactically
+                       (lambda ()
+                         (re-search-forward
+                          python-nav-beginning-of-defun-regexp nil t))
+                       '>)))
+      (setq arg (1- arg)))
+    (while (and (< arg 0)
+                (setq found
+                      (python-nav--syntactically
+                       (lambda ()
+                         (re-search-backward
+                          python-nav-beginning-of-defun-regexp nil t))
+                       '<)))
+      (setq arg (1+ arg)))
+    found))
+
+(defun python-nav-backward-defun (&optional arg)
+  "Navigate to closer defun backward ARG times.
+Unlikely `python-nav-beginning-of-defun' this doesn't care about
+nested definitions."
+  (interactive "^p")
+  (python-nav--forward-defun (- (or arg 1))))
+
+(defun python-nav-forward-defun (&optional arg)
+  "Navigate to closer defun forward ARG times.
+Unlikely `python-nav-beginning-of-defun' this doesn't care about
+nested definitions."
+  (interactive "^p")
+  (python-nav--forward-defun (or arg 1)))
+

 (defun python-nav-beginning-of-statement ()
   "Move to start of current statement."
   (interactive "^")
 (defun python-nav-beginning-of-statement ()
   "Move to start of current statement."
   (interactive "^")

-  (while (and (or (back-to-indentation) t)
-              (not (bobp))
-              (when (or
-                     (save-excursion
-                       (forward-line -1)
-                       (python-info-line-ends-backslash-p))
-                     (python-syntax-context 'string)
-                     (python-syntax-context 'paren))
-                (forward-line -1))))
+  (back-to-indentation)
+  (let* ((ppss (syntax-ppss))
+         (context-point
+          (or
+           (python-syntax-context 'paren ppss)
+           (python-syntax-context 'string ppss))))
+    (cond ((bobp))
+          (context-point
+           (goto-char context-point)
+           (python-nav-beginning-of-statement))
+          ((save-excursion
+             (forward-line -1)
+             (python-info-line-ends-backslash-p))
+           (forward-line -1)
+           (python-nav-beginning-of-statement))))

   (point-marker))
 
 (defun python-nav-end-of-statement (&optional noend)
   (point-marker))
 
 (defun python-nav-end-of-statement (&optional noend)


@@ -1253,9 +1405,7 @@ backward to previous statement."
 (defun python-nav-beginning-of-block ()
   "Move to start of current block."
   (interactive "^")
 (defun python-nav-beginning-of-block ()
   "Move to start of current block."
   (interactive "^")

-  (let ((starting-pos (point))
-        (block-regexp (python-rx
-                       line-start (* whitespace) block-start)))
+  (let ((starting-pos (point)))

     (if (progn
           (python-nav-beginning-of-statement)
           (looking-at (python-rx block-start)))
     (if (progn
           (python-nav-beginning-of-statement)
           (looking-at (python-rx block-start)))


@@ -1324,33 +1474,41 @@ backward to previous block."
         (and (goto-char starting-pos) nil)
       (and (not (= (point) starting-pos)) (point-marker)))))
 
         (and (goto-char starting-pos) nil)
       (and (not (= (point) starting-pos)) (point-marker)))))
 

-(defun python-nav-lisp-forward-sexp-safe (&optional arg)
-  "Safe version of standard `forward-sexp'.
-When ARG > 0 move forward, else if ARG is < 0."
-  (or arg (setq arg 1))
+(defun python-nav--lisp-forward-sexp (&optional arg)
+  "Standard version `forward-sexp'.
+It ignores completely the value of `forward-sexp-function' by
+setting it to nil before calling `forward-sexp'.  With positive
+ARG move forward only one sexp, else move backwards."

   (let ((forward-sexp-function)
   (let ((forward-sexp-function)

-        (paren-regexp
-         (if (> arg 0) (python-rx close-paren) (python-rx open-paren)))
-        (search-fn
-         (if (> arg 0) #'re-search-forward #'re-search-backward)))
+        (arg (if (or (not arg) (> arg 0)) 1 -1)))
+    (forward-sexp arg)))
+
+(defun python-nav--lisp-forward-sexp-safe (&optional arg)
+  "Safe version of standard `forward-sexp'.
+When at end of sexp (i.e. looking at a opening/closing paren)
+skips it instead of throwing an error.  With positive ARG move
+forward only one sexp, else move backwards."
+  (let* ((arg (if (or (not arg) (> arg 0)) 1 -1))
+         (paren-regexp
+          (if (> arg 0) (python-rx close-paren) (python-rx open-paren)))
+         (search-fn
+          (if (> arg 0) #'re-search-forward #'re-search-backward)))

     (condition-case nil
     (condition-case nil

-        (forward-sexp arg)
+        (python-nav--lisp-forward-sexp arg)

       (error
        (while (and (funcall search-fn paren-regexp nil t)
                    (python-syntax-context 'paren)))))))
 
       (error
        (while (and (funcall search-fn paren-regexp nil t)
                    (python-syntax-context 'paren)))))))
 

-(defun python-nav--forward-sexp (&optional dir)
+(defun python-nav--forward-sexp (&optional dir safe)

   "Move to forward sexp.
   "Move to forward sexp.

-With positive Optional argument DIR direction move forward, else
-backwards."
+With positive optional argument DIR direction move forward, else
+backwards.  When optional argument SAFE is non-nil do not throw
+errors when at end of sexp, skip it instead."

   (setq dir (or dir 1))
   (unless (= dir 0)
     (let* ((forward-p (if (> dir 0)
                           (and (setq dir 1) t)
                         (and (setq dir -1) nil)))
   (setq dir (or dir 1))
   (unless (= dir 0)
     (let* ((forward-p (if (> dir 0)
                           (and (setq dir 1) t)
                         (and (setq dir -1) nil)))

-           (re-search-fn (if forward-p
-                             're-search-forward
-                           're-search-backward))

            (context-type (python-syntax-context-type)))
       (cond
        ((memq context-type '(string comment))
            (context-type (python-syntax-context-type)))
       (cond
        ((memq context-type '(string comment))


@@ -1363,7 +1521,9 @@ backwards."
                  (eq (syntax-class (syntax-after (1- (point))))
                      (car (string-to-syntax ")")))))
         ;; Inside a paren or looking at it, lisp knows what to do.
                  (eq (syntax-class (syntax-after (1- (point))))
                      (car (string-to-syntax ")")))))
         ;; Inside a paren or looking at it, lisp knows what to do.

-        (python-nav-lisp-forward-sexp-safe dir))
+        (if safe
+            (python-nav--lisp-forward-sexp-safe dir)
+          (python-nav--lisp-forward-sexp dir)))

        (t
         ;; This part handles the lispy feel of
         ;; `python-nav-forward-sexp'.  Knowing everything about the
        (t
         ;; This part handles the lispy feel of
         ;; `python-nav-forward-sexp'.  Knowing everything about the


@@ -1377,7 +1537,9 @@ backwards."
                  ((python-info-end-of-statement-p) 'statement-end)))
                (next-sexp-pos
                 (save-excursion
                  ((python-info-end-of-statement-p) 'statement-end)))
                (next-sexp-pos
                 (save-excursion

-                  (python-nav-lisp-forward-sexp-safe dir)
+                  (if safe
+                      (python-nav--lisp-forward-sexp-safe dir)
+                    (python-nav--lisp-forward-sexp dir))

                   (point)))
                (next-sexp-context
                 (save-excursion
                   (point)))
                (next-sexp-context
                 (save-excursion


@@ -1431,23 +1593,48 @@ backwards."
                    (python-nav-beginning-of-statement))
                   (t (goto-char next-sexp-pos))))))))))
 
                    (python-nav-beginning-of-statement))
                   (t (goto-char next-sexp-pos))))))))))
 

-(defun python-nav--backward-sexp ()
-  "Move to backward sexp."
-  (python-nav--forward-sexp -1))
-

 (defun python-nav-forward-sexp (&optional arg)
 (defun python-nav-forward-sexp (&optional arg)

-  "Move forward across one block of code.
-With ARG, do it that many times.  Negative arg -N means
-move backward N times."
+  "Move forward across expressions.
+With ARG, do it that many times.  Negative arg -N means move
+backward N times."
+  (interactive "^p")
+  (or arg (setq arg 1))
+  (while (> arg 0)
+    (python-nav--forward-sexp 1)
+    (setq arg (1- arg)))
+  (while (< arg 0)
+    (python-nav--forward-sexp -1)
+    (setq arg (1+ arg))))
+
+(defun python-nav-backward-sexp (&optional arg)
+  "Move backward across expressions.
+With ARG, do it that many times.  Negative arg -N means move
+forward N times."
+  (interactive "^p")
+  (or arg (setq arg 1))
+  (python-nav-forward-sexp (- arg)))
+
+(defun python-nav-forward-sexp-safe (&optional arg)
+  "Move forward safely across expressions.
+With ARG, do it that many times.  Negative arg -N means move
+backward N times."

   (interactive "^p")
   (or arg (setq arg 1))
   (while (> arg 0)
   (interactive "^p")
   (or arg (setq arg 1))
   (while (> arg 0)

-    (python-nav--forward-sexp)
+    (python-nav--forward-sexp 1 t)

     (setq arg (1- arg)))
   (while (< arg 0)
     (setq arg (1- arg)))
   (while (< arg 0)

-    (python-nav--backward-sexp)
+    (python-nav--forward-sexp -1 t)

     (setq arg (1+ arg))))
 
     (setq arg (1+ arg))))
 

+(defun python-nav-backward-sexp-safe (&optional arg)
+  "Move backward safely across expressions.
+With ARG, do it that many times.  Negative arg -N means move
+forward N times."
+  (interactive "^p")
+  (or arg (setq arg 1))
+  (python-nav-forward-sexp-safe (- arg)))
+

 (defun python-nav--up-list (&optional dir)
   "Internal implementation of `python-nav-up-list'.
 DIR is always 1 or -1 and comes sanitized from
 (defun python-nav--up-list (&optional dir)
   "Internal implementation of `python-nav-up-list'.
 DIR is always 1 or -1 and comes sanitized from


@@ -1503,12 +1690,35 @@ This command assumes point is not in a string or comment."
 (defun python-nav-backward-up-list (&optional arg)
   "Move backward out of one level of parentheses (or blocks).
 With ARG, do this that many times.
 (defun python-nav-backward-up-list (&optional arg)
   "Move backward out of one level of parentheses (or blocks).
 With ARG, do this that many times.

-A negative argument means move backward but still to a less deep spot.
+A negative argument means move forward but still to a less deep spot.

 This command assumes point is not in a string or comment."
   (interactive "^p")
   (or arg (setq arg 1))
   (python-nav-up-list (- arg)))
 
 This command assumes point is not in a string or comment."
   (interactive "^p")
   (or arg (setq arg 1))
   (python-nav-up-list (- arg)))
 

+(defun python-nav-if-name-main ()
+  "Move point at the beginning the __main__ block.
+When \"if __name__ == '__main__':\" is found returns its
+position, else returns nil."
+  (interactive)
+  (let ((point (point))
+        (found (catch 'found
+                 (goto-char (point-min))
+                 (while (re-search-forward
+                         (python-rx line-start
+                                    "if" (+ space)
+                                    "__name__" (+ space)
+                                    "==" (+ space)
+                                    (group-n 1 (or ?\" ?\'))
+                                    "__main__" (backref 1) (* space) ":")
+                         nil t)
+                   (when (not (python-syntax-context-type))
+                     (beginning-of-line)
+                     (throw 'found t))))))
+    (if found
+        (point)
+      (ignore (goto-char point)))))
+

 \f
 ;;; Shell integration
 
 \f
 ;;; Shell integration
 


@@ -1534,37 +1744,68 @@ This command assumes point is not in a string or comment."
   :type 'string
   :group 'python)
 
   :type 'string
   :group 'python)
 

+(defcustom python-shell-interpreter-interactive-arg "-i"
+  "Interpreter argument to force it to run interactively."
+  :type 'string
+  :version "24.4")
+
+(defcustom python-shell-prompt-detect-enabled t
+  "Non-nil enables autodetection of interpreter prompts."
+  :type 'boolean
+  :safe 'booleanp
+  :version "24.4")
+
+(defcustom python-shell-prompt-detect-failure-warning t
+  "Non-nil enables warnings when detection of prompts fail."
+  :type 'boolean
+  :safe 'booleanp
+  :version "24.4")
+
+(defcustom python-shell-prompt-input-regexps
+  '(">>> " "\\.\\.\\. "                 ; Python
+    "In \\[[0-9]+\\]: "                 ; IPython
+    "   \\.\\.\\.: "                    ; IPython
+    ;; Using ipdb outside IPython may fail to cleanup and leave static
+    ;; IPython prompts activated, this adds some safeguard for that.
+    "In : " "\\.\\.\\.: ")
+  "List of regular expressions matching input prompts."
+  :type '(repeat string)
+  :version "24.4")
+
+(defcustom python-shell-prompt-output-regexps
+  '(""                                  ; Python
+    "Out\\[[0-9]+\\]: "                 ; IPython
+    "Out :")                            ; ipdb safeguard
+  "List of regular expressions matching output prompts."
+  :type '(repeat string)
+  :version "24.4")
+

 (defcustom python-shell-prompt-regexp ">>> "
 (defcustom python-shell-prompt-regexp ">>> "

-  "Regular Expression matching top\-level input prompt of python shell.
+  "Regular expression matching top level input prompt of Python shell.

 It should not contain a caret (^) at the beginning."
 It should not contain a caret (^) at the beginning."

-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)

 
 

-(defcustom python-shell-prompt-block-regexp "[.][.][.] "
-  "Regular Expression matching block input prompt of python shell.
+(defcustom python-shell-prompt-block-regexp "\\.\\.\\. "
+  "Regular expression matching block input prompt of Python shell.

 It should not contain a caret (^) at the beginning."
 It should not contain a caret (^) at the beginning."

-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)

 
 (defcustom python-shell-prompt-output-regexp ""
 
 (defcustom python-shell-prompt-output-regexp ""

-  "Regular Expression matching output prompt of python shell.
+  "Regular expression matching output prompt of Python shell.

 It should not contain a caret (^) at the beginning."
 It should not contain a caret (^) at the beginning."

-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)

 
 (defcustom python-shell-prompt-pdb-regexp "[(<]*[Ii]?[Pp]db[>)]+ "
 
 (defcustom python-shell-prompt-pdb-regexp "[(<]*[Ii]?[Pp]db[>)]+ "

-  "Regular Expression matching pdb input prompt of python shell.
+  "Regular expression matching pdb input prompt of Python shell.

 It should not contain a caret (^) at the beginning."
 It should not contain a caret (^) at the beginning."

-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)
+
+(define-obsolete-variable-alias
+  'python-shell-enable-font-lock 'python-shell-font-lock-enable "25.1")

 
 

-(defcustom python-shell-enable-font-lock t
-  "Should syntax highlighting be enabled in the python shell buffer?
-Restart the python shell after changing this variable for it to take effect."
+(defcustom python-shell-font-lock-enable t
+  "Should syntax highlighting be enabled in the Python shell buffer?
+Restart the Python shell after changing this variable for it to take effect."

   :type 'boolean
   :group 'python
   :safe 'booleanp)
   :type 'boolean
   :group 'python
   :safe 'booleanp)


@@ -1603,7 +1844,7 @@ This variable, when set to a string, makes the values stored in
 `python-shell-process-environment' and `python-shell-exec-path'
 to be modified properly so shells are started with the specified
 virtualenv."
 `python-shell-process-environment' and `python-shell-exec-path'
 to be modified properly so shells are started with the specified
 virtualenv."

-  :type 'string
+  :type '(choice (const nil) string)

   :group 'python
   :safe 'stringp)
 
   :group 'python
   :safe 'stringp)
 


@@ -1630,6 +1871,169 @@ virtualenv."
   :type '(alist string)
   :group 'python)
 
   :type '(alist string)
   :group 'python)
 

+(defvar python-shell--prompt-calculated-input-regexp nil
+  "Calculated input prompt regexp for inferior python shell.
+Do not set this variable directly, instead use
+`python-shell-prompt-set-calculated-regexps'.")
+
+(defvar python-shell--prompt-calculated-output-regexp nil
+  "Calculated output prompt regexp for inferior python shell.
+Do not set this variable directly, instead use
+`python-shell-set-prompt-regexp'.")
+
+(defun python-shell-prompt-detect ()
+  "Detect prompts for the current `python-shell-interpreter'.
+When prompts can be retrieved successfully from the
+`python-shell-interpreter' run with
+`python-shell-interpreter-interactive-arg', returns a list of
+three elements, where the first two are input prompts and the
+last one is an output prompt.  When no prompts can be detected
+and `python-shell-prompt-detect-failure-warning' is non-nil,
+shows a warning with instructions to avoid hangs and returns nil.
+When `python-shell-prompt-detect-enabled' is nil avoids any
+detection and just returns nil."
+  (when python-shell-prompt-detect-enabled
+    (let* ((process-environment (python-shell-calculate-process-environment))
+           (exec-path (python-shell-calculate-exec-path))
+           (code (concat
+                  "import sys\n"
+                  "ps = [getattr(sys, 'ps%s' % i, '') for i in range(1,4)]\n"
+                  ;; JSON is built manually for compatibility
+                  "ps_json = '\\n[\"%s\", \"%s\", \"%s\"]\\n' % tuple(ps)\n"
+                  "print (ps_json)\n"
+                  "sys.exit(0)\n"))
+           (output
+            (with-temp-buffer
+              ;; TODO: improve error handling by using
+              ;; `condition-case' and displaying the error message to
+              ;; the user in the no-prompts warning.
+              (ignore-errors
+                (let ((code-file (python-shell--save-temp-file code)))
+                  ;; Use `process-file' as it is remote-host friendly.
+                  (process-file
+                   python-shell-interpreter
+                   code-file
+                   '(t nil)
+                   nil
+                   python-shell-interpreter-interactive-arg)
+                  ;; Try to cleanup
+                  (delete-file code-file)))
+              (buffer-string)))
+           (prompts
+            (catch 'prompts
+              (dolist (line (split-string output "\n" t))
+                (let ((res
+                       ;; Check if current line is a valid JSON array
+                       (and (string= (substring line 0 2) "[\"")
+                            (ignore-errors
+                              ;; Return prompts as a list, not vector
+                              (append (json-read-from-string line) nil)))))
+                  ;; The list must contain 3 strings, where the first
+                  ;; is the input prompt, the second is the block
+                  ;; prompt and the last one is the output prompt.  The
+                  ;; input prompt is the only one that can't be empty.
+                  (when (and (= (length res) 3)
+                             (cl-every #'stringp res)
+                             (not (string= (car res) "")))
+                    (throw 'prompts res))))
+              nil)))
+      (when (and (not prompts)
+                 python-shell-prompt-detect-failure-warning)
+        (lwarn
+         '(python python-shell-prompt-regexp)
+         :warning
+         (concat
+          "Python shell prompts cannot be detected.\n"
+          "If your emacs session hangs when starting python shells\n"
+          "recover with `keyboard-quit' and then try fixing the\n"
+          "interactive flag for your interpreter by adjusting the\n"
+          "`python-shell-interpreter-interactive-arg' or add regexps\n"
+          "matching shell prompts in the directory-local friendly vars:\n"
+          "  + `python-shell-prompt-regexp'\n"
+          "  + `python-shell-prompt-block-regexp'\n"
+          "  + `python-shell-prompt-output-regexp'\n"
+          "Or alternatively in:\n"
+          "  + `python-shell-prompt-input-regexps'\n"
+          "  + `python-shell-prompt-output-regexps'")))
+      prompts)))
+
+(defun python-shell-prompt-validate-regexps ()
+  "Validate all user provided regexps for prompts.
+Signals `user-error' if any of these vars contain invalid
+regexps: `python-shell-prompt-regexp',
+`python-shell-prompt-block-regexp',
+`python-shell-prompt-pdb-regexp',
+`python-shell-prompt-output-regexp',
+`python-shell-prompt-input-regexps',
+`python-shell-prompt-output-regexps'."
+  (dolist (symbol (list 'python-shell-prompt-input-regexps
+                        'python-shell-prompt-output-regexps
+                        'python-shell-prompt-regexp
+                        'python-shell-prompt-block-regexp
+                        'python-shell-prompt-pdb-regexp
+                        'python-shell-prompt-output-regexp))
+    (dolist (regexp (let ((regexps (symbol-value symbol)))
+                      (if (listp regexps)
+                          regexps
+                        (list regexps))))
+      (when (not (python-util-valid-regexp-p regexp))
+        (user-error "Invalid regexp %s in `%s'"
+                    regexp symbol)))))
+
+(defun python-shell-prompt-set-calculated-regexps ()
+  "Detect and set input and output prompt regexps.
+Build and set the values for `python-shell-input-prompt-regexp'
+and `python-shell-output-prompt-regexp' using the values from
+`python-shell-prompt-regexp', `python-shell-prompt-block-regexp',
+`python-shell-prompt-pdb-regexp',
+`python-shell-prompt-output-regexp',
+`python-shell-prompt-input-regexps',
+`python-shell-prompt-output-regexps' and detected prompts from
+`python-shell-prompt-detect'."
+  (when (not (and python-shell--prompt-calculated-input-regexp
+                  python-shell--prompt-calculated-output-regexp))
+    (let* ((detected-prompts (python-shell-prompt-detect))
+           (input-prompts nil)
+           (output-prompts nil)
+           (build-regexp
+            (lambda (prompts)
+              (concat "^\\("
+                      (mapconcat #'identity
+                                 (sort prompts
+                                       (lambda (a b)
+                                         (let ((length-a (length a))
+                                               (length-b (length b)))
+                                           (if (= length-a length-b)
+                                               (string< a b)
+                                             (> (length a) (length b))))))
+                                 "\\|")
+                      "\\)"))))
+      ;; Validate ALL regexps
+      (python-shell-prompt-validate-regexps)
+      ;; Collect all user defined input prompts
+      (dolist (prompt (append python-shell-prompt-input-regexps
+                              (list python-shell-prompt-regexp
+                                    python-shell-prompt-block-regexp
+                                    python-shell-prompt-pdb-regexp)))
+        (cl-pushnew prompt input-prompts :test #'string=))
+      ;; Collect all user defined output prompts
+      (dolist (prompt (cons python-shell-prompt-output-regexp
+                            python-shell-prompt-output-regexps))
+        (cl-pushnew prompt output-prompts :test #'string=))
+      ;; Collect detected prompts if any
+      (when detected-prompts
+        (dolist (prompt (butlast detected-prompts))
+          (setq prompt (regexp-quote prompt))
+          (cl-pushnew prompt input-prompts :test #'string=))
+        (cl-pushnew (regexp-quote
+                     (car (last detected-prompts)))
+                    output-prompts :test #'string=))
+      ;; Set input and output prompt regexps from collected prompts
+      (setq python-shell--prompt-calculated-input-regexp
+            (funcall build-regexp input-prompts)
+            python-shell--prompt-calculated-output-regexp
+            (funcall build-regexp output-prompts)))))
+

 (defun python-shell-get-process-name (dedicated)
   "Calculate the appropriate process name for inferior Python process.
 If DEDICATED is t and the variable `buffer-file-name' is non-nil
 (defun python-shell-get-process-name (dedicated)
   "Calculate the appropriate process name for inferior Python process.
 If DEDICATED is t and the variable `buffer-file-name' is non-nil


@@ -1652,10 +2056,10 @@ uniqueness for different types of configurations."
           python-shell-internal-buffer-name
           (md5
            (concat
           python-shell-internal-buffer-name
           (md5
            (concat

-            (python-shell-parse-command)
-            python-shell-prompt-regexp
-            python-shell-prompt-block-regexp
-            python-shell-prompt-output-regexp
+            python-shell-interpreter
+            python-shell-interpreter-args
+            python-shell--prompt-calculated-input-regexp
+            python-shell--prompt-calculated-output-regexp

             (mapconcat #'symbol-value python-shell-setup-codes "")
             (mapconcat #'identity python-shell-process-environment "")
             (mapconcat #'identity python-shell-extra-pythonpaths "")
             (mapconcat #'symbol-value python-shell-setup-codes "")
             (mapconcat #'identity python-shell-process-environment "")
             (mapconcat #'identity python-shell-extra-pythonpaths "")


@@ -1663,14 +2067,27 @@ uniqueness for different types of configurations."
             (or python-shell-virtualenv-path "")
             (mapconcat #'identity python-shell-exec-path "")))))
 
             (or python-shell-virtualenv-path "")
             (mapconcat #'identity python-shell-exec-path "")))))
 

-(defun python-shell-parse-command ()
+(defun python-shell-parse-command ()    ;FIXME: why name it "parse"?

   "Calculate the string used to execute the inferior Python process."
   "Calculate the string used to execute the inferior Python process."

+  ;; FIXME: process-environment doesn't seem to be used anywhere within
+  ;; this let.

   (let ((process-environment (python-shell-calculate-process-environment))
         (exec-path (python-shell-calculate-exec-path)))
     (format "%s %s"
   (let ((process-environment (python-shell-calculate-process-environment))
         (exec-path (python-shell-calculate-exec-path)))
     (format "%s %s"

+            ;; FIXME: Why executable-find?

             (executable-find python-shell-interpreter)
             python-shell-interpreter-args)))
 
             (executable-find python-shell-interpreter)
             python-shell-interpreter-args)))
 

+(defun python-new-pythonpath ()
+  "Calculate the new PYTHONPATH value from `python-shell-extra-pythonpaths'."
+  (let ((pythonpath (getenv "PYTHONPATH"))
+        (extra (mapconcat 'identity
+                          python-shell-extra-pythonpaths
+                          path-separator)))
+    (if pythonpath
+        (concat extra path-separator pythonpath)
+      extra)))
+

 (defun python-shell-calculate-process-environment ()
   "Calculate process environment given `python-shell-virtualenv-path'."
   (let ((process-environment (append
 (defun python-shell-calculate-process-environment ()
   "Calculate process environment given `python-shell-virtualenv-path'."
   (let ((process-environment (append


@@ -1680,13 +2097,7 @@ uniqueness for different types of configurations."
                         (directory-file-name python-shell-virtualenv-path)
                       nil)))
     (when python-shell-extra-pythonpaths
                         (directory-file-name python-shell-virtualenv-path)
                       nil)))
     (when python-shell-extra-pythonpaths

-      (setenv "PYTHONPATH"
-              (format "%s%s%s"
-                      (mapconcat 'identity
-                                 python-shell-extra-pythonpaths
-                                 path-separator)
-                      path-separator
-                      (or (getenv "PYTHONPATH") ""))))
+      (setenv "PYTHONPATH" (python-new-pythonpath)))

     (if (not virtualenv)
         process-environment
       (setenv "PYTHONHOME" nil)
     (if (not virtualenv)
         process-environment
       (setenv "PYTHONHOME" nil)


@@ -1699,109 +2110,311 @@ uniqueness for different types of configurations."
 (defun python-shell-calculate-exec-path ()
   "Calculate exec path given `python-shell-virtualenv-path'."
   (let ((path (append python-shell-exec-path
 (defun python-shell-calculate-exec-path ()
   "Calculate exec path given `python-shell-virtualenv-path'."
   (let ((path (append python-shell-exec-path

-                      exec-path nil)))
+                      exec-path nil)))  ;FIXME: Why nil?

     (if (not python-shell-virtualenv-path)
         path
     (if (not python-shell-virtualenv-path)
         path

-      (cons (format "%s/bin"
-                    (directory-file-name python-shell-virtualenv-path))
+      (cons (expand-file-name "bin" python-shell-virtualenv-path)

             path))))
 
             path))))
 

-(defun python-comint-output-filter-function (output)
-  "Hook run after content is put into comint buffer.
-OUTPUT is a string with the contents of the buffer."
-  (ansi-color-filter-apply output))
+(defvar python-shell--package-depth 10)
+
+(defun python-shell-package-enable (directory package)
+  "Add DIRECTORY parent to $PYTHONPATH and enable PACKAGE."
+  (interactive
+   (let* ((dir (expand-file-name
+                (read-directory-name
+                 "Package root: "
+                 (file-name-directory
+                  (or (buffer-file-name) default-directory)))))
+          (name (completing-read
+                 "Package: "
+                 (python-util-list-packages
+                  dir python-shell--package-depth))))
+     (list dir name)))
+  (python-shell-send-string
+   (format
+    (concat
+     "import os.path;import sys;"
+     "sys.path.append(os.path.dirname(os.path.dirname('''%s''')));"
+     "__package__ = '''%s''';"
+     "import %s")
+    directory package package)
+   (python-shell-get-process)))
+
+(defun python-shell-accept-process-output (process &optional timeout regexp)
+  "Accept PROCESS output with TIMEOUT until REGEXP is found.
+Optional argument TIMEOUT is the timeout argument to
+`accept-process-output' calls.  Optional argument REGEXP
+overrides the regexp to match the end of output, defaults to
+`comint-prompt-regexp.'.  Returns non-nil when output was
+properly captured.
+
+This utility is useful in situations where the output may be
+received in chunks, since `accept-process-output' gives no
+guarantees they will be grabbed in a single call.  An example use
+case for this would be the CPython shell start-up, where the
+banner and the initial prompt are received separately."
+  (let ((regexp (or regexp comint-prompt-regexp)))
+    (catch 'found
+      (while t
+        (when (not (accept-process-output process timeout))
+          (throw 'found nil))
+        (when (looking-back regexp)
+          (throw 'found t))))))
+
+(defun python-shell-comint-end-of-output-p (output)
+  "Return non-nil if OUTPUT is ends with input prompt."
+  (string-match
+   ;; XXX: It seems on OSX an extra carriage return is attached
+   ;; at the end of output, this handles that too.
+   (concat
+    "\r?\n?"
+    ;; Remove initial caret from calculated regexp
+    (replace-regexp-in-string
+     (rx string-start ?^) ""
+     python-shell--prompt-calculated-input-regexp)
+    (rx eos))
+   output))
+
+(define-obsolete-function-alias
+  'python-comint-output-filter-function
+  'ansi-color-filter-apply
+  "25.1")
+
+(defun python-comint-postoutput-scroll-to-bottom (output)
+  "Faster version of `comint-postoutput-scroll-to-bottom'.
+Avoids `recenter' calls until OUTPUT is completely sent."
+  (when (and (not (string= "" output))
+             (python-shell-comint-end-of-output-p
+              (ansi-color-filter-apply output)))
+    (comint-postoutput-scroll-to-bottom output))
+  output)

 
 (defvar python-shell--parent-buffer nil)
 
 
 (defvar python-shell--parent-buffer nil)
 

-(defvar python-shell-output-syntax-table
-  (let ((table (make-syntax-table python-dotty-syntax-table)))
-    (modify-syntax-entry ?\' "." table)
-    (modify-syntax-entry ?\" "." table)
-    (modify-syntax-entry ?\( "." table)
-    (modify-syntax-entry ?\[ "." table)
-    (modify-syntax-entry ?\{ "." table)
-    (modify-syntax-entry ?\) "." table)
-    (modify-syntax-entry ?\] "." table)
-    (modify-syntax-entry ?\} "." table)
-    table)
-  "Syntax table for shell output.
-It makes parens and quotes be treated as punctuation chars.")
+(defmacro python-shell-with-shell-buffer (&rest body)
+  "Execute the forms in BODY with the shell buffer temporarily current.
+Signals an error if no shell buffer is available for current buffer."
+  (declare (indent 0) (debug t))
+  (let ((shell-buffer (make-symbol "shell-buffer")))
+    `(let ((,shell-buffer (python-shell-get-buffer)))
+       (when (not ,shell-buffer)
+         (error "No inferior Python buffer available."))
+       (with-current-buffer ,shell-buffer
+         ,@body))))
+
+(defvar python-shell--font-lock-buffer nil)
+
+(defun python-shell-font-lock-get-or-create-buffer ()
+  "Get or create a font-lock buffer for current inferior process."
+  (python-shell-with-shell-buffer
+    (if python-shell--font-lock-buffer
+        python-shell--font-lock-buffer
+      (let ((process-name
+             (process-name (get-buffer-process (current-buffer)))))
+        (generate-new-buffer
+         (format "*%s-font-lock*" process-name))))))
+
+(defun python-shell-font-lock-kill-buffer ()
+  "Kill the font-lock buffer safely."
+  (python-shell-with-shell-buffer
+    (when (and python-shell--font-lock-buffer
+               (buffer-live-p python-shell--font-lock-buffer))
+      (kill-buffer python-shell--font-lock-buffer)
+      (when (eq major-mode 'inferior-python-mode)
+        (setq python-shell--font-lock-buffer nil)))))
+
+(defmacro python-shell-font-lock-with-font-lock-buffer (&rest body)
+  "Execute the forms in BODY in the font-lock buffer.
+The value returned is the value of the last form in BODY.  See
+also `with-current-buffer'."
+  (declare (indent 0) (debug t))
+  `(python-shell-with-shell-buffer
+     (save-current-buffer
+       (when (not (and python-shell--font-lock-buffer
+                       (get-buffer python-shell--font-lock-buffer)))
+         (setq python-shell--font-lock-buffer
+               (python-shell-font-lock-get-or-create-buffer)))
+       (set-buffer python-shell--font-lock-buffer)
+       (set (make-local-variable 'delay-mode-hooks) t)
+       (let ((python-indent-guess-indent-offset nil))
+         (when (not (eq major-mode 'python-mode))
+           (python-mode))
+         ,@body))))
+
+(defun python-shell-font-lock-cleanup-buffer ()
+  "Cleanup the font-lock buffer.
+Provided as a command because this might be handy if something
+goes wrong and syntax highlighting in the shell gets messed up."
+  (interactive)
+  (python-shell-with-shell-buffer
+    (python-shell-font-lock-with-font-lock-buffer
+      (delete-region (point-min) (point-max)))))
+
+(defun python-shell-font-lock-comint-output-filter-function (output)
+  "Clean up the font-lock buffer after any OUTPUT."
+  (when (and (not (string= "" output))
+             ;; Is end of output and is not just a prompt.
+             (not (member
+                   (python-shell-comint-end-of-output-p
+                    (ansi-color-filter-apply output))
+                   '(nil 0))))
+    ;; If output is other than an input prompt then "real" output has
+    ;; been received and the font-lock buffer must be cleaned up.
+    (python-shell-font-lock-cleanup-buffer))
+  output)
+
+(defun python-shell-font-lock-post-command-hook ()
+  "Fontifies current line in shell buffer."
+  (if (eq this-command 'comint-send-input)
+      ;; Add a newline when user sends input as this may be a block.
+      (python-shell-font-lock-with-font-lock-buffer
+        (goto-char (line-end-position))
+        (newline))
+    (when (and (python-util-comint-last-prompt)
+               (> (point) (cdr (python-util-comint-last-prompt))))
+      (let ((input (buffer-substring-no-properties
+                    (cdr (python-util-comint-last-prompt)) (point-max)))
+            (old-input (python-shell-font-lock-with-font-lock-buffer
+                         (buffer-substring-no-properties
+                          (line-beginning-position) (point-max))))
+            (current-point (point))
+            (buffer-undo-list t))
+        ;; When input hasn't changed, do nothing.
+        (when (not (string= input old-input))
+          (delete-region (cdr (python-util-comint-last-prompt)) (point-max))
+          (insert
+           (python-shell-font-lock-with-font-lock-buffer
+             (delete-region (line-beginning-position)
+                            (line-end-position))
+             (insert input)
+             ;; Ensure buffer is fontified, keeping it
+             ;; compatible with Emacs < 24.4.
+             (if (fboundp 'font-lock-ensure)
+                 (funcall 'font-lock-ensure)
+               (font-lock-default-fontify-buffer))
+             ;; Replace FACE text properties with FONT-LOCK-FACE so
+             ;; they are not overwritten by comint buffer's font lock.
+             (python-util-text-properties-replace-name
+              'face 'font-lock-face)
+             (buffer-substring (line-beginning-position)
+                               (line-end-position))))
+          (goto-char current-point))))))
+
+(defun python-shell-font-lock-turn-on (&optional msg)
+  "Turn on shell font-lock.
+With argument MSG show activation message."
+  (interactive "p")
+  (python-shell-with-shell-buffer
+    (python-shell-font-lock-kill-buffer)
+    (set (make-local-variable 'python-shell--font-lock-buffer) nil)
+    (add-hook 'post-command-hook
+              #'python-shell-font-lock-post-command-hook nil 'local)
+    (add-hook 'kill-buffer-hook
+              #'python-shell-font-lock-kill-buffer nil 'local)
+    (add-hook 'comint-output-filter-functions
+              #'python-shell-font-lock-comint-output-filter-function
+              'append 'local)
+    (when msg
+      (message "Shell font-lock is enabled"))))
+
+(defun python-shell-font-lock-turn-off (&optional msg)
+  "Turn off shell font-lock.
+With argument MSG show deactivation message."
+  (interactive "p")
+  (python-shell-with-shell-buffer
+    (python-shell-font-lock-kill-buffer)
+    (when (python-util-comint-last-prompt)
+      ;; Cleanup current fontification
+      (remove-text-properties
+       (cdr (python-util-comint-last-prompt))
+       (line-end-position)
+       '(face nil font-lock-face nil)))
+    (set (make-local-variable 'python-shell--font-lock-buffer) nil)
+    (remove-hook 'post-command-hook
+                 #'python-shell-font-lock-post-command-hook'local)
+    (remove-hook 'kill-buffer-hook
+                 #'python-shell-font-lock-kill-buffer 'local)
+    (remove-hook 'comint-output-filter-functions
+                 #'python-shell-font-lock-comint-output-filter-function
+                 'local)
+    (when msg
+      (message "Shell font-lock is disabled"))))
+
+(defun python-shell-font-lock-toggle (&optional msg)
+  "Toggle font-lock for shell.
+With argument MSG show activation/deactivation message."
+  (interactive "p")
+  (python-shell-with-shell-buffer
+    (set (make-local-variable 'python-shell-font-lock-enable)
+         (not python-shell-font-lock-enable))
+    (if python-shell-font-lock-enable
+        (python-shell-font-lock-turn-on msg)
+      (python-shell-font-lock-turn-off msg))
+    python-shell-font-lock-enable))

 
 (define-derived-mode inferior-python-mode comint-mode "Inferior Python"
   "Major mode for Python inferior process.
 Runs a Python interpreter as a subprocess of Emacs, with Python
 
 (define-derived-mode inferior-python-mode comint-mode "Inferior Python"
   "Major mode for Python inferior process.
 Runs a Python interpreter as a subprocess of Emacs, with Python

-I/O through an Emacs buffer.  Variables
-`python-shell-interpreter' and `python-shell-interpreter-args'
-controls which Python interpreter is run.  Variables
+I/O through an Emacs buffer.  Variables `python-shell-interpreter'
+and `python-shell-interpreter-args' control which Python
+interpreter is run.  Variables

 `python-shell-prompt-regexp',
 `python-shell-prompt-output-regexp',
 `python-shell-prompt-block-regexp',
 `python-shell-prompt-regexp',
 `python-shell-prompt-output-regexp',
 `python-shell-prompt-block-regexp',

-`python-shell-enable-font-lock',
+`python-shell-font-lock-enable',

 `python-shell-completion-setup-code',
 `python-shell-completion-string-code',
 `python-shell-completion-setup-code',
 `python-shell-completion-string-code',

-`python-shell-completion-module-string-code',

 `python-eldoc-setup-code', `python-eldoc-string-code',
 `python-ffap-setup-code' and `python-ffap-string-code' can
 customize this mode for different Python interpreters.
 
 `python-eldoc-setup-code', `python-eldoc-string-code',
 `python-ffap-setup-code' and `python-ffap-string-code' can
 customize this mode for different Python interpreters.
 

+This mode resets `comint-output-filter-functions' locally, so you
+may want to re-add custom functions to it using the
+`inferior-python-mode-hook'.
+

 You can also add additional setup code to be run at
 initialization of the interpreter via `python-shell-setup-codes'
 variable.
 
 \(Type \\[describe-mode] in the process buffer for a list of commands.)"
 You can also add additional setup code to be run at
 initialization of the interpreter via `python-shell-setup-codes'
 variable.
 
 \(Type \\[describe-mode] in the process buffer for a list of commands.)"

-  (and python-shell--parent-buffer
-       (python-util-clone-local-variables python-shell--parent-buffer))
-  (setq comint-prompt-regexp (format "^\\(?:%s\\|%s\\|%s\\)"
-                                     python-shell-prompt-regexp
-                                     python-shell-prompt-block-regexp
-                                     python-shell-prompt-pdb-regexp))
+  (let ((interpreter python-shell-interpreter)
+        (args python-shell-interpreter-args))
+    (when python-shell--parent-buffer
+      (python-util-clone-local-variables python-shell--parent-buffer))
+    ;; Users can override default values for these vars when calling
+    ;; `run-python'.  This ensures new values let-bound in
+    ;; `python-shell-make-comint' are locally set.
+    (set (make-local-variable 'python-shell-interpreter) interpreter)
+    (set (make-local-variable 'python-shell-interpreter-args) args))
+  (set (make-local-variable 'python-shell--prompt-calculated-input-regexp) nil)
+  (set (make-local-variable 'python-shell--prompt-calculated-output-regexp) nil)
+  (python-shell-prompt-set-calculated-regexps)
+  (setq comint-prompt-regexp python-shell--prompt-calculated-input-regexp
+        comint-prompt-read-only t)

   (setq mode-line-process '(":%s"))
   (setq mode-line-process '(":%s"))

-  (make-local-variable 'comint-output-filter-functions)
-  (add-hook 'comint-output-filter-functions
-            'python-comint-output-filter-function)
-  (add-hook 'comint-output-filter-functions
-            'python-pdbtrack-comint-output-filter-function)
+  (set (make-local-variable 'comint-output-filter-functions)
+       '(ansi-color-process-output
+         python-pdbtrack-comint-output-filter-function
+         python-comint-postoutput-scroll-to-bottom))

   (set (make-local-variable 'compilation-error-regexp-alist)
        python-shell-compilation-regexp-alist)
   (set (make-local-variable 'compilation-error-regexp-alist)
        python-shell-compilation-regexp-alist)

-  (define-key inferior-python-mode-map [remap complete-symbol]
-    'completion-at-point)

   (add-hook 'completion-at-point-functions
   (add-hook 'completion-at-point-functions

-            'python-shell-completion-complete-at-point nil 'local)
-  (add-to-list (make-local-variable 'comint-dynamic-complete-functions)
-               'python-shell-completion-complete-at-point)
+            #'python-shell-completion-at-point nil 'local)

   (define-key inferior-python-mode-map "\t"
     'python-shell-completion-complete-or-indent)
   (make-local-variable 'python-pdbtrack-buffers-to-kill)
   (make-local-variable 'python-pdbtrack-tracked-buffer)
   (make-local-variable 'python-shell-internal-last-output)
   (define-key inferior-python-mode-map "\t"
     'python-shell-completion-complete-or-indent)
   (make-local-variable 'python-pdbtrack-buffers-to-kill)
   (make-local-variable 'python-pdbtrack-tracked-buffer)
   (make-local-variable 'python-shell-internal-last-output)

-  (when python-shell-enable-font-lock
-    (set-syntax-table python-mode-syntax-table)
-    (set (make-local-variable 'font-lock-defaults)
-         '(python-font-lock-keywords nil nil nil nil))
-    (set (make-local-variable 'syntax-propertize-function)
-         (eval
-          ;; XXX: Unfortunately eval is needed here to make use of the
-          ;; dynamic value of `comint-prompt-regexp'.
-          `(syntax-propertize-rules
-            (,comint-prompt-regexp
-             (0 (ignore
-                 (put-text-property
-                  comint-last-input-start end 'syntax-table
-                  python-shell-output-syntax-table)
-                 ;; XXX: This might look weird, but it is the easiest
-                 ;; way to ensure font lock gets cleaned up before the
-                 ;; current prompt, which is needed for unclosed
-                 ;; strings to not mess up with current input.
-                 (font-lock-unfontify-region comint-last-input-start end))))
-            (,(python-rx string-delimiter)
-             (0 (ignore
-                 (and (not (eq (get-text-property start 'field) 'output))
-                      (python-syntax-stringify)))))))))
-  (compilation-shell-minor-mode 1))
+  (when python-shell-font-lock-enable
+    (python-shell-font-lock-turn-on))
+  (compilation-shell-minor-mode 1)
+  (python-shell-accept-process-output
+   (get-buffer-process (current-buffer))))

 
 (defun python-shell-make-comint (cmd proc-name &optional pop internal)
 
 (defun python-shell-make-comint (cmd proc-name &optional pop internal)

-  "Create a python shell comint buffer.
-CMD is the python command to be executed and PROC-NAME is the
+  "Create a Python shell comint buffer.
+CMD is the Python command to be executed and PROC-NAME is the

 process name the comint buffer will get.  After the comint buffer
 is created the `inferior-python-mode' is activated.  When
 optional argument POP is non-nil the buffer is shown.  When
 process name the comint buffer will get.  After the comint buffer
 is created the `inferior-python-mode' is activated.  When
 optional argument POP is non-nil the buffer is shown.  When


@@ -1817,13 +2430,22 @@ killed."
            (exec-path (python-shell-calculate-exec-path)))
       (when (not (comint-check-proc proc-buffer-name))
         (let* ((cmdlist (split-string-and-unquote cmd))
            (exec-path (python-shell-calculate-exec-path)))
       (when (not (comint-check-proc proc-buffer-name))
         (let* ((cmdlist (split-string-and-unquote cmd))

+               (interpreter (car cmdlist))
+               (args (cdr cmdlist))

                (buffer (apply #'make-comint-in-buffer proc-name proc-buffer-name
                (buffer (apply #'make-comint-in-buffer proc-name proc-buffer-name

-                              (car cmdlist) nil (cdr cmdlist)))
+                              interpreter nil args))

                (python-shell--parent-buffer (current-buffer))
                (python-shell--parent-buffer (current-buffer))

-               (process (get-buffer-process buffer)))
+               (process (get-buffer-process buffer))
+               ;; As the user may have overridden default values for
+               ;; these vars on `run-python', let-binding them allows
+               ;; to have the new right values in all setup code
+               ;; that's is done in `inferior-python-mode', which is
+               ;; important, especially for prompt detection.
+               (python-shell-interpreter interpreter)
+               (python-shell-interpreter-args
+                (mapconcat #'identity args " ")))

           (with-current-buffer buffer
             (inferior-python-mode))
           (with-current-buffer buffer
             (inferior-python-mode))

-          (accept-process-output process)

           (and pop (pop-to-buffer buffer t))
           (and internal (set-process-query-on-exit-flag process nil))))
       proc-buffer-name)))
           (and pop (pop-to-buffer buffer t))
           (and internal (set-process-query-on-exit-flag process nil))))
       proc-buffer-name)))


@@ -1840,13 +2462,13 @@ command used to call the interpreter and define DEDICATED, so a
 dedicated process for the current buffer is open.  When numeric
 prefix arg is other than 0 or 4 do not SHOW.
 
 dedicated process for the current buffer is open.  When numeric
 prefix arg is other than 0 or 4 do not SHOW.
 

-Runs the hook `inferior-python-mode-hook' (after the
-`comint-mode-hook' is run).  \(Type \\[describe-mode] in the
+Runs the hook `inferior-python-mode-hook' after
+`comint-mode-hook' is run.  (Type \\[describe-mode] in the

 process buffer for a list of commands.)"
   (interactive
    (if current-prefix-arg
        (list
 process buffer for a list of commands.)"
   (interactive
    (if current-prefix-arg
        (list

-        (read-string "Run Python: " (python-shell-parse-command))
+        (read-shell-command "Run Python: " (python-shell-parse-command))

         (y-or-n-p "Make dedicated process? ")
         (= (prefix-numeric-value current-prefix-arg) 4))
      (list (python-shell-parse-command) nil t)))
         (y-or-n-p "Make dedicated process? ")
         (= (prefix-numeric-value current-prefix-arg) 4))
      (list (python-shell-parse-command) nil t)))


@@ -1861,50 +2483,54 @@ Input and output via buffer named after
 `python-shell-internal-get-process-name' returns.
 
 This new kind of shell is intended to be used for generic
 `python-shell-internal-get-process-name' returns.
 
 This new kind of shell is intended to be used for generic

-communication related to defined configurations, the main
+communication related to defined configurations; the main

 difference with global or dedicated shells is that these ones are
 attached to a configuration, not a buffer.  This means that can
 be used for example to retrieve the sys.path and other stuff,
 without messing with user shells.  Note that
 difference with global or dedicated shells is that these ones are
 attached to a configuration, not a buffer.  This means that can
 be used for example to retrieve the sys.path and other stuff,
 without messing with user shells.  Note that

-`python-shell-enable-font-lock' and `inferior-python-mode-hook'
+`python-shell-font-lock-enable' and `inferior-python-mode-hook'

 are set to nil for these shells, so setup codes are not sent at
 startup."
 are set to nil for these shells, so setup codes are not sent at
 startup."

-  (let ((python-shell-enable-font-lock nil)
+  (let ((python-shell-font-lock-enable nil)

         (inferior-python-mode-hook nil))
     (get-buffer-process
      (python-shell-make-comint
       (python-shell-parse-command)
       (python-shell-internal-get-process-name) nil t))))
 
         (inferior-python-mode-hook nil))
     (get-buffer-process
      (python-shell-make-comint
       (python-shell-parse-command)
       (python-shell-internal-get-process-name) nil t))))
 

+(defun python-shell-get-buffer ()
+  "Return inferior Python buffer for current buffer.
+If current buffer is in `inferior-python-mode', return it."
+  (if (eq major-mode 'inferior-python-mode)
+      (current-buffer)
+    (let* ((dedicated-proc-name (python-shell-get-process-name t))
+           (dedicated-proc-buffer-name (format "*%s*" dedicated-proc-name))
+           (global-proc-name  (python-shell-get-process-name nil))
+           (global-proc-buffer-name (format "*%s*" global-proc-name))
+           (dedicated-running (comint-check-proc dedicated-proc-buffer-name))
+           (global-running (comint-check-proc global-proc-buffer-name)))
+      ;; Always prefer dedicated
+      (or (and dedicated-running dedicated-proc-buffer-name)
+          (and global-running global-proc-buffer-name)))))
+

 (defun python-shell-get-process ()
 (defun python-shell-get-process ()

-  "Get inferior Python process for current buffer and return it."
-  (let* ((dedicated-proc-name (python-shell-get-process-name t))
-         (dedicated-proc-buffer-name (format "*%s*" dedicated-proc-name))
-         (global-proc-name  (python-shell-get-process-name nil))
-         (global-proc-buffer-name (format "*%s*" global-proc-name))
-         (dedicated-running (comint-check-proc dedicated-proc-buffer-name))
-         (global-running (comint-check-proc global-proc-buffer-name)))
-    ;; Always prefer dedicated
-    (get-buffer-process (or (and dedicated-running dedicated-proc-buffer-name)
-                            (and global-running global-proc-buffer-name)))))
-
-(defun python-shell-get-or-create-process ()
-  "Get or create an inferior Python process for current buffer and return it."
-  (let* ((dedicated-proc-name (python-shell-get-process-name t))
-         (dedicated-proc-buffer-name (format "*%s*" dedicated-proc-name))
-         (global-proc-name  (python-shell-get-process-name nil))
-         (global-proc-buffer-name (format "*%s*" global-proc-name))
-         (dedicated-running (comint-check-proc dedicated-proc-buffer-name))
-         (global-running (comint-check-proc global-proc-buffer-name))
-         (current-prefix-arg 16))
-    (when (and (not dedicated-running) (not global-running))
-      (if (call-interactively 'run-python)
-          (setq dedicated-running t)
-        (setq global-running t)))
-    ;; Always prefer dedicated
-    (get-buffer-process (if dedicated-running
-                            dedicated-proc-buffer-name
-                          global-proc-buffer-name))))
+  "Return inferior Python process for current buffer."
+  (get-buffer-process (python-shell-get-buffer)))
+
+(defun python-shell-get-or-create-process (&optional cmd dedicated show)
+  "Get or create an inferior Python process for current buffer and return it.
+Arguments CMD, DEDICATED and SHOW are those of `run-python' and
+are used to start the shell.  If those arguments are not
+provided, `run-python' is called interactively and the user will
+be asked for their values."
+  (let ((shell-process (python-shell-get-process)))
+    (when (not shell-process)
+      (if (not cmd)
+          ;; XXX: Refactor code such that calling `run-python'
+          ;; interactively is not needed anymore.
+          (call-interactively 'run-python)
+        (run-python cmd dedicated show)))
+    (or shell-process (python-shell-get-process))))

 
 (defvar python-shell-internal-buffer nil
   "Current internal shell buffer for the current buffer.
 
 (defvar python-shell-internal-buffer nil
   "Current internal shell buffer for the current buffer.


@@ -1922,13 +2548,7 @@ there for compatibility with CEDET.")
          (proc-buffer-name (format " *%s*" proc-name)))
     (when (not (process-live-p proc-name))
       (run-python-internal)
          (proc-buffer-name (format " *%s*" proc-name)))
     (when (not (process-live-p proc-name))
       (run-python-internal)

-      (setq python-shell-internal-buffer proc-buffer-name)
-      ;; XXX: Why is this `sit-for' needed?
-      ;; `python-shell-make-comint' calls `accept-process-output'
-      ;; already but it is not helping to get proper output on
-      ;; 'gnu/linux when the internal shell process is not running and
-      ;; a call to `python-shell-internal-send-string' is issued.
-      (sit-for 0.1 t))
+      (setq python-shell-internal-buffer proc-buffer-name))

     (get-buffer-process proc-buffer-name)))
 
 (define-obsolete-function-alias
     (get-buffer-process proc-buffer-name)))
 
 (define-obsolete-function-alias


@@ -1940,27 +2560,29 @@ there for compatibility with CEDET.")
 (define-obsolete-variable-alias
   'python-preoutput-result 'python-shell-internal-last-output "24.3")
 
 (define-obsolete-variable-alias
   'python-preoutput-result 'python-shell-internal-last-output "24.3")
 

-(defun python-shell-send-string (string &optional process msg)
-  "Send STRING to inferior Python PROCESS.
-When MSG is non-nil messages the first line of STRING."
+(defun python-shell--save-temp-file (string)
+  (let* ((temporary-file-directory
+          (if (file-remote-p default-directory)
+              (concat (file-remote-p default-directory) "/tmp")
+            temporary-file-directory))
+         (temp-file-name (make-temp-file "py"))
+         (coding-system-for-write 'utf-8))
+    (with-temp-file temp-file-name
+      (insert "# -*- coding: utf-8 -*-\n") ;Not needed for Python-3.
+      (insert string)
+      (delete-trailing-whitespace))
+    temp-file-name))
+
+(defun python-shell-send-string (string &optional process)
+  "Send STRING to inferior Python PROCESS."

   (interactive "sPython command: ")
   (interactive "sPython command: ")

-  (let ((process (or process (python-shell-get-or-create-process)))
-        (lines (split-string string "\n" t)))
-    (and msg (message "Sent: %s..." (nth 0 lines)))
-    (if (> (length lines) 1)
-        (let* ((temporary-file-directory
-                (if (file-remote-p default-directory)
-                    (concat (file-remote-p default-directory) "/tmp")
-                  temporary-file-directory))
-               (temp-file-name (make-temp-file "py"))
-               (file-name (or (buffer-file-name) temp-file-name)))
-          (with-temp-file temp-file-name
-            (insert string)
-            (delete-trailing-whitespace))
-          (python-shell-send-file file-name process temp-file-name))
+  (let ((process (or process (python-shell-get-or-create-process))))
+    (if (string-match ".\n+." string)   ;Multiline.
+        (let* ((temp-file-name (python-shell--save-temp-file string)))
+          (python-shell-send-file temp-file-name process temp-file-name t))

       (comint-send-string process string)
       (comint-send-string process string)

-      (when (or (not (string-match "\n$" string))
-                (string-match "\n[ \t].*\n?$" string))
+      (when (or (not (string-match "\n\\'" string))
+                (string-match "\n[ \t].*\n?\\'" string))

         (comint-send-string process "\n")))))
 
 (defvar python-shell-output-filter-in-progress nil)
         (comint-send-string process "\n")))))
 
 (defvar python-shell-output-filter-in-progress nil)


@@ -1976,13 +2598,7 @@ detecting a prompt at the end of the buffer."
    string (ansi-color-filter-apply string)
    python-shell-output-filter-buffer
    (concat python-shell-output-filter-buffer string))
    string (ansi-color-filter-apply string)
    python-shell-output-filter-buffer
    (concat python-shell-output-filter-buffer string))

-  (when (string-match
-         ;; XXX: It seems on OSX an extra carriage return is attached
-         ;; at the end of output, this handles that too.
-         (format "\r?\n\\(?:%s\\|%s\\|%s\\)$"
-                 python-shell-prompt-regexp
-                 python-shell-prompt-block-regexp
-                 python-shell-prompt-pdb-regexp)
+  (when (python-shell-comint-end-of-output-p

          python-shell-output-filter-buffer)
     ;; Output ends when `python-shell-output-filter-buffer' contains
     ;; the prompt attached at the end of it.
          python-shell-output-filter-buffer)
     ;; Output ends when `python-shell-output-filter-buffer' contains
     ;; the prompt attached at the end of it.


@@ -1990,19 +2606,18 @@ detecting a prompt at the end of the buffer."
           python-shell-output-filter-buffer
           (substring python-shell-output-filter-buffer
                      0 (match-beginning 0)))
           python-shell-output-filter-buffer
           (substring python-shell-output-filter-buffer
                      0 (match-beginning 0)))

-    (when (and (> (length python-shell-prompt-output-regexp) 0)
-               (string-match (concat "^" python-shell-prompt-output-regexp)
-                             python-shell-output-filter-buffer))
-      ;; Some shells, like iPython might append a prompt before the
+    (when (string-match
+           python-shell--prompt-calculated-output-regexp
+           python-shell-output-filter-buffer)
+      ;; Some shells, like IPython might append a prompt before the

       ;; output, clean that.
       (setq python-shell-output-filter-buffer
             (substring python-shell-output-filter-buffer (match-end 0)))))
   "")
 
       ;; output, clean that.
       (setq python-shell-output-filter-buffer
             (substring python-shell-output-filter-buffer (match-end 0)))))
   "")
 

-(defun python-shell-send-string-no-output (string &optional process msg)
+(defun python-shell-send-string-no-output (string &optional process)

   "Send STRING to PROCESS and inhibit output.
   "Send STRING to PROCESS and inhibit output.

-When MSG is non-nil messages the first line of STRING.  Return
-the output."
+Return the output."

   (let ((process (or process (python-shell-get-or-create-process)))
         (comint-preoutput-filter-functions
          '(python-shell-output-filter))
   (let ((process (or process (python-shell-get-or-create-process)))
         (comint-preoutput-filter-functions
          '(python-shell-output-filter))


@@ -2010,7 +2625,7 @@ the output."
         (inhibit-quit t))
     (or
      (with-local-quit
         (inhibit-quit t))
     (or
      (with-local-quit

-       (python-shell-send-string string process msg)
+       (python-shell-send-string string process)

        (while python-shell-output-filter-in-progress
          ;; `python-shell-output-filter' takes care of setting
          ;; `python-shell-output-filter-in-progress' to NIL after it
        (while python-shell-output-filter-in-progress
          ;; `python-shell-output-filter' takes care of setting
          ;; `python-shell-output-filter-in-progress' to NIL after it


@@ -2032,7 +2647,7 @@ Returns the output.  See `python-shell-send-string-no-output'."
          ;; Makes this function compatible with the old
          ;; python-send-receive. (At least for CEDET).
          (replace-regexp-in-string "_emacs_out +" "" string)
          ;; Makes this function compatible with the old
          ;; python-send-receive. (At least for CEDET).
          (replace-regexp-in-string "_emacs_out +" "" string)

-         (python-shell-internal-get-or-create-process) nil)))
+         (python-shell-internal-get-or-create-process))))

 
 (define-obsolete-function-alias
   'python-send-receive 'python-shell-internal-send-string "24.3")
 
 (define-obsolete-function-alias
   'python-send-receive 'python-shell-internal-send-string "24.3")


@@ -2040,32 +2655,94 @@ Returns the output.  See `python-shell-send-string-no-output'."
 (define-obsolete-function-alias
   'python-send-string 'python-shell-internal-send-string "24.3")
 
 (define-obsolete-function-alias
   'python-send-string 'python-shell-internal-send-string "24.3")
 

-(defun python-shell-send-region (start end)
+(defvar python--use-fake-loc nil
+  "If non-nil, use `compilation-fake-loc' to trace errors back to the buffer.
+If nil, regions of text are prepended by the corresponding number of empty
+lines and Python is told to output error messages referring to the whole
+source file.")
+
+(defun python-shell-buffer-substring (start end &optional nomain)
+  "Send buffer substring from START to END formatted for shell.
+This is a wrapper over `buffer-substring' that takes care of
+different transformations for the code sent to be evaluated in
+the python shell:
+  1. When optional argument NOMAIN is non-nil everything under an
+     \"if __name__ == '__main__'\" block will be removed.
+  2. When a subregion of the buffer is sent, it takes care of
+     appending extra empty lines so tracebacks are correct.
+  3. Wraps indented regions under an \"if True:\" block so the
+     interpreter evaluates them correctly."
+  (let ((substring (buffer-substring-no-properties start end))
+        (fillstr (unless python--use-fake-loc
+                   (make-string (1- (line-number-at-pos start)) ?\n)))
+        (toplevel-block-p (save-excursion
+                            (goto-char start)
+                            (or (zerop (line-number-at-pos start))
+                                (progn
+                                  (python-util-forward-comment 1)
+                                  (zerop (current-indentation)))))))
+    (with-temp-buffer
+      (python-mode)
+      (if fillstr (insert fillstr))
+      (insert substring)
+      (goto-char (point-min))
+      (unless python--use-fake-loc
+        ;; python-shell--save-temp-file adds an extra coding line, which would
+        ;; throw off the line-counts, so let's try to compensate here.
+        (if (looking-at "[ \t]*[#\n]")
+            (delete-region (point) (line-beginning-position 2))))
+      (when (not toplevel-block-p)
+        (insert "if True:")
+        (delete-region (point) (line-end-position)))
+      (when nomain
+        (let* ((if-name-main-start-end
+                (and nomain
+                     (save-excursion
+                       (when (python-nav-if-name-main)
+                         (cons (point)
+                               (progn (python-nav-forward-sexp-safe)
+                                      (point)))))))
+               ;; Oh destructuring bind, how I miss you.
+               (if-name-main-start (car if-name-main-start-end))
+               (if-name-main-end (cdr if-name-main-start-end)))
+          (when if-name-main-start-end
+            (goto-char if-name-main-start)
+            (delete-region if-name-main-start if-name-main-end)
+            (insert
+             (make-string
+              (- (line-number-at-pos if-name-main-end)
+                 (line-number-at-pos if-name-main-start)) ?\n)))))
+      (buffer-substring-no-properties (point-min) (point-max)))))
+
+(declare-function compilation-fake-loc "compile"
+                  (marker file &optional line col))
+
+(defun python-shell-send-region (start end &optional nomain)

   "Send the region delimited by START and END to inferior Python process."
   (interactive "r")
   "Send the region delimited by START and END to inferior Python process."
   (interactive "r")

-  (python-shell-send-string
-   (concat
-    (let ((line-num (line-number-at-pos start)))
-      ;; When sending a region, add blank lines for non sent code so
-      ;; backtraces remain correct.
-      (make-string (1- line-num) ?\n))
-    (buffer-substring start end))
-   nil t))
+  (let* ((python--use-fake-loc
+          (or python--use-fake-loc (not buffer-file-name)))
+         (string (python-shell-buffer-substring start end nomain))
+         (process (python-shell-get-or-create-process))
+         (_ (string-match "\\`\n*\\(.*\\)" string)))
+    (message "Sent: %s..." (match-string 1 string))
+    (let* ((temp-file-name (python-shell--save-temp-file string))
+           (file-name (or (buffer-file-name) temp-file-name)))
+      (python-shell-send-file file-name process temp-file-name t)
+      (unless python--use-fake-loc
+        (with-current-buffer (process-buffer process)
+          (compilation-fake-loc (copy-marker start) temp-file-name
+                                2)) ;; Not 1, because of the added coding line.
+        ))))

 
 (defun python-shell-send-buffer (&optional arg)
   "Send the entire buffer to inferior Python process.
 With prefix ARG allow execution of code inside blocks delimited
 
 (defun python-shell-send-buffer (&optional arg)
   "Send the entire buffer to inferior Python process.
 With prefix ARG allow execution of code inside blocks delimited

-by \"if __name__== '__main__':\""
+by \"if __name__== '__main__':\"."

   (interactive "P")
   (save-restriction
     (widen)
   (interactive "P")
   (save-restriction
     (widen)

-    (let ((str (buffer-substring (point-min) (point-max))))
-      (and
-       (not arg)
-       (setq str (replace-regexp-in-string
-                  (python-rx if-name-main)
-                  "if __name__ == '__main__ ':" str)))
-      (python-shell-send-string str))))
+    (python-shell-send-region (point-min) (point-max) (not arg))))

 
 (defun python-shell-send-defun (arg)
   "Send the current defun to inferior Python process.
 
 (defun python-shell-send-defun (arg)
   "Send the current defun to inferior Python process.


@@ -2088,11 +2765,12 @@ When argument ARG is non-nil do not include decorators."
            (end-of-line 1))
        (point-marker)))))
 
            (end-of-line 1))
        (point-marker)))))
 

-(defun python-shell-send-file (file-name &optional process temp-file-name)
+(defun python-shell-send-file (file-name &optional process temp-file-name
+                                         delete)

   "Send FILE-NAME to inferior Python PROCESS.
 If TEMP-FILE-NAME is passed then that file is used for processing
   "Send FILE-NAME to inferior Python PROCESS.
 If TEMP-FILE-NAME is passed then that file is used for processing

-instead, while internally the shell will continue to use
-FILE-NAME."
+instead, while internally the shell will continue to use FILE-NAME.
+If DELETE is non-nil, delete the file afterwards."

   (interactive "fFile to send: ")
   (let* ((process (or process (python-shell-get-or-create-process)))
          (temp-file-name (when temp-file-name
   (interactive "fFile to send: ")
   (let* ((process (or process (python-shell-get-or-create-process)))
          (temp-file-name (when temp-file-name


@@ -2110,25 +2788,34 @@ FILE-NAME."
      (format
       (concat "__pyfile = open('''%s''');"
               "exec(compile(__pyfile.read(), '''%s''', 'exec'));"
      (format
       (concat "__pyfile = open('''%s''');"
               "exec(compile(__pyfile.read(), '''%s''', 'exec'));"

-              "__pyfile.close()")
-      (or temp-file-name file-name) file-name)
+              "__pyfile.close()%s")
+      (or temp-file-name file-name) file-name
+      (if delete (format "; import os; os.remove('''%s''')"
+                         (or temp-file-name file-name))
+        ""))

      process)))
 
 (defun python-shell-switch-to-shell ()
   "Switch to inferior Python process buffer."
   (interactive)
      process)))
 
 (defun python-shell-switch-to-shell ()
   "Switch to inferior Python process buffer."
   (interactive)

-  (pop-to-buffer (process-buffer (python-shell-get-or-create-process)) t))
+  (process-buffer (python-shell-get-or-create-process)) t)

 
 (defun python-shell-send-setup-code ()
   "Send all setup code for shell.
 This function takes the list of setup code to send from the
 `python-shell-setup-codes' list."
 
 (defun python-shell-send-setup-code ()
   "Send all setup code for shell.
 This function takes the list of setup code to send from the
 `python-shell-setup-codes' list."

-  (let ((process (get-buffer-process (current-buffer))))
-    (dolist (code python-shell-setup-codes)
-      (when code
-        (message "Sent %s" code)
-        (python-shell-send-string
-         (symbol-value code) process)))))
+  (let ((process (python-shell-get-process))
+        (code (concat
+               (mapconcat
+                (lambda (elt)
+                  (cond ((stringp elt) elt)
+                        ((symbolp elt) (symbol-value elt))
+                        (t "")))
+                python-shell-setup-codes
+                "\n\n")
+               "\n\nprint ('python.el: sent setup code')")))
+    (python-shell-send-string code process)
+    (python-shell-accept-process-output process)))

 
 (add-hook 'inferior-python-mode-hook
           #'python-shell-send-setup-code)
 
 (add-hook 'inferior-python-mode-hook
           #'python-shell-send-setup-code)


@@ -2138,23 +2825,35 @@ This function takes the list of setup code to send from the
 
 (defcustom python-shell-completion-setup-code
   "try:
 
 (defcustom python-shell-completion-setup-code
   "try:

-    import readline
+    import readline, rlcompleter

 except ImportError:
 except ImportError:

-    def __COMPLETER_all_completions(text): []
+    def __PYTHON_EL_get_completions(text):
+        return []

 else:
 else:

-    import rlcompleter
-    readline.set_completer(rlcompleter.Completer().complete)
-    def __COMPLETER_all_completions(text):
-        import sys
+    def __PYTHON_EL_get_completions(text):

         completions = []
         try:
         completions = []
         try:

-            i = 0
-            while True:
-                res = readline.get_completer()(text, i)
-                if not res: break
-                i += 1
-                completions.append(res)
-        except NameError:
+            splits = text.split()
+            is_module = splits and splits[0] in ('from', 'import')
+            is_ipython = getattr(
+                __builtins__, '__IPYTHON__',
+                getattr(__builtins__, '__IPYTHON__active', False))
+            if is_module:
+                from IPython.core.completerlib import module_completion
+                completions = module_completion(text.strip())
+            elif is_ipython and getattr(__builtins__, '__IP', None):
+                completions = __IP.complete(text)
+            elif is_ipython and getattr(__builtins__, 'get_ipython', None):
+                completions = get_ipython().Completer.all_completions(text)
+            else:
+                i = 0
+                while True:
+                    res = readline.get_completer()(text, i)
+                    if not res:
+                        break
+                    i += 1
+                    completions.append(res)
+        except:

             pass
         return completions"
   "Code used to setup completion in inferior Python processes."
             pass
         return completions"
   "Code used to setup completion in inferior Python processes."


@@ -2162,122 +2861,106 @@ else:
   :group 'python)
 
 (defcustom python-shell-completion-string-code
   :group 'python)
 
 (defcustom python-shell-completion-string-code

-  "';'.join(__COMPLETER_all_completions('''%s'''))\n"
-  "Python code used to get a string of completions separated by semicolons."
-  :type 'string
-  :group 'python)
-
-(defcustom python-shell-completion-module-string-code ""
-  "Python code used to get completions separated by semicolons for imports.
-
-For IPython v0.11, add the following line to
-`python-shell-completion-setup-code':
-
-from IPython.core.completerlib import module_completion
-
-and use the following as the value of this variable:
-
-';'.join(module_completion('''%s'''))\n"
+  "';'.join(__PYTHON_EL_get_completions('''%s'''))\n"
+  "Python code used to get a string of completions separated by semicolons.
+The string passed to the function is the current python name or
+the full statement in the case of imports."

   :type 'string
   :group 'python)
 
   :type 'string
   :group 'python)
 

-(defcustom python-shell-completion-pdb-string-code
-  "';'.join(globals().keys() + locals().keys())"
-  "Python code used to get completions separated by semicolons for [i]pdb."
-  :type 'string
-  :group 'python)
+(define-obsolete-variable-alias
+  'python-shell-completion-module-string-code
+  'python-shell-completion-string-code
+  "24.4"
+  "Completion string code must also autocomplete modules.")

 
 

-(defun python-shell-completion-get-completions (process line input)
-  "Do completion at point for PROCESS.
-LINE is used to detect the context on how to complete given
-INPUT."
-  (let* ((prompt
-          ;; Get the last prompt for the inferior process
-          ;; buffer. This is used for the completion code selection
-          ;; heuristic.
-          (with-current-buffer (process-buffer process)
-            (buffer-substring-no-properties
-             (overlay-start comint-last-prompt-overlay)
-             (overlay-end comint-last-prompt-overlay))))
-         (completion-context
-          ;; Check whether a prompt matches a pdb string, an import
-          ;; statement or just the standard prompt and use the
-          ;; correct python-shell-completion-*-code string
-          (cond ((and (> (length python-shell-completion-pdb-string-code) 0)
-                      (string-match
-                       (concat "^" python-shell-prompt-pdb-regexp) prompt))
-                 'pdb)
-                ((and (>
-                       (length python-shell-completion-module-string-code) 0)
-                      (string-match
-                       (concat "^" python-shell-prompt-regexp) prompt)
-                      (string-match "^[ \t]*\\(from\\|import\\)[ \t]" line))
-                 'import)
-                ((string-match
-                  (concat "^" python-shell-prompt-regexp) prompt)
-                 'default)
-                (t nil)))
-         (completion-code
-          (pcase completion-context
-            (`pdb python-shell-completion-pdb-string-code)
-            (`import python-shell-completion-module-string-code)
-            (`default python-shell-completion-string-code)
-            (_ nil)))
-         (input
-          (if (eq completion-context 'import)
-              (replace-regexp-in-string "^[ \t]+" "" line)
-            input)))
-    (and completion-code
-         (> (length input) 0)
-         (with-current-buffer (process-buffer process)
-           (let ((completions (python-shell-send-string-no-output
-                               (format completion-code input) process)))
+(define-obsolete-variable-alias
+  'python-shell-completion-pdb-string-code
+  'python-shell-completion-string-code
+  "25.1"
+  "Completion string code must work for (i)pdb.")
+
+(defun python-shell-completion-get-completions (process import input)
+  "Do completion at point using PROCESS for IMPORT or INPUT.
+When IMPORT is non-nil takes precedence over INPUT for
+completion."
+  (with-current-buffer (process-buffer process)
+    (let* ((prompt
+            ;; Get last prompt of the inferior process buffer (this
+            ;; intentionally avoids using `comint-last-prompt' because
+            ;; of incompatibilities with Emacs 24.x).
+            (save-excursion
+              (buffer-substring-no-properties
+               (line-beginning-position) ;End of prompt.
+               (re-search-backward "^"))))
+           (completion-code
+            ;; Check whether a prompt matches a pdb string, an import
+            ;; statement or just the standard prompt and use the
+            ;; correct python-shell-completion-*-code string
+            (cond ((and (string-match
+                         (concat "^" python-shell-prompt-pdb-regexp) prompt))
+                   ;; Since there are no guarantees the user will remain
+                   ;; in the same context where completion code was sent
+                   ;; (e.g. user steps into a function), safeguard
+                   ;; resending completion setup continuously.
+                   (concat python-shell-completion-setup-code
+                           "\nprint (" python-shell-completion-string-code ")"))
+                  ((string-match
+                    python-shell--prompt-calculated-input-regexp prompt)
+                   python-shell-completion-string-code)
+                  (t nil)))
+           (subject (or import input)))
+      (and completion-code
+           (> (length input) 0)
+           (let ((completions
+                  (python-util-strip-string
+                   (python-shell-send-string-no-output
+                    (format completion-code subject) process))))

              (and (> (length completions) 2)
                   (split-string completions
                                 "^'\\|^\"\\|;\\|'$\\|\"$" t)))))))
 
              (and (> (length completions) 2)
                   (split-string completions
                                 "^'\\|^\"\\|;\\|'$\\|\"$" t)))))))
 

-(defun python-shell-completion-complete-at-point (&optional process)
-  "Perform completion at point in inferior Python.
+(defun python-shell-completion-at-point (&optional process)
+  "Function for `completion-at-point-functions' in `inferior-python-mode'.

 Optional argument PROCESS forces completions to be retrieved
 using that one instead of current buffer's process."
   (setq process (or process (get-buffer-process (current-buffer))))
 Optional argument PROCESS forces completions to be retrieved
 using that one instead of current buffer's process."
   (setq process (or process (get-buffer-process (current-buffer))))

-  (let* ((start
+  (let* ((last-prompt-end (cdr (python-util-comint-last-prompt)))
+         (import-statement
+          (when (string-match-p
+                 (rx (* space) word-start (or "from" "import") word-end space)
+                 (buffer-substring-no-properties last-prompt-end (point)))
+            (buffer-substring-no-properties last-prompt-end (point))))
+         (start

           (save-excursion
           (save-excursion

-            (with-syntax-table python-dotty-syntax-table
-              (let* ((paren-depth (car (syntax-ppss)))
-                     (syntax-string "w_")
-                     (syntax-list (string-to-syntax syntax-string)))
-                ;; Stop scanning for the beginning of the completion
-                ;; subject after the char before point matches a
-                ;; delimiter
-                (while (member
-                        (car (syntax-after (1- (point)))) syntax-list)
-                  (skip-syntax-backward syntax-string)
-                  (when (or (equal (char-before) ?\))
-                            (equal (char-before) ?\"))
-                    (forward-char -1))
-                  (while (or
-                          ;; honor initial paren depth
-                          (> (car (syntax-ppss)) paren-depth)
-                          (python-syntax-context 'string))
-                    (forward-char -1)))
-                (point)))))
+            (if (not (re-search-backward
+                      (python-rx
+                       (or whitespace open-paren close-paren string-delimiter))
+                      last-prompt-end
+                      t 1))
+                last-prompt-end
+              (forward-char (length (match-string-no-properties 0)))
+              (point))))

          (end (point)))
     (list start end
           (completion-table-dynamic
            (apply-partially
             #'python-shell-completion-get-completions
          (end (point)))
     (list start end
           (completion-table-dynamic
            (apply-partially
             #'python-shell-completion-get-completions

-            process (buffer-substring-no-properties
-                     (line-beginning-position) end))))))
+            process import-statement)))))
+
+(define-obsolete-function-alias
+  'python-shell-completion-complete-at-point
+  'python-shell-completion-at-point
+  "25.1")

 
 (defun python-shell-completion-complete-or-indent ()
   "Complete or indent depending on the context.
 
 (defun python-shell-completion-complete-or-indent ()
   "Complete or indent depending on the context.

-If content before pointer is all whitespace indent.  If not try
-to complete."
+If content before pointer is all whitespace, indent.
+If not try to complete."

   (interactive)
   (if (string-match "^[[:space:]]*$"
                     (buffer-substring (comint-line-beginning-position)
   (interactive)
   (if (string-match "^[[:space:]]*$"
                     (buffer-substring (comint-line-beginning-position)

-                                      (point-marker)))
+                                      (point)))

       (indent-for-tab-command)
     (completion-at-point)))
 
       (indent-for-tab-command)
     (completion-at-point)))
 


@@ -2285,14 +2968,14 @@ to complete."
 ;;; PDB Track integration
 
 (defcustom python-pdbtrack-activate t
 ;;; PDB Track integration
 
 (defcustom python-pdbtrack-activate t

-  "Non-nil makes python shell enable pdbtracking."
+  "Non-nil makes Python shell enable pdbtracking."

   :type 'boolean
   :group 'python
   :safe 'booleanp)
 
 (defcustom python-pdbtrack-stacktrace-info-regexp
   :type 'boolean
   :group 'python
   :safe 'booleanp)
 
 (defcustom python-pdbtrack-stacktrace-info-regexp

-  "^> \\([^\"(<]+\\)(\\([0-9]+\\))\\([?a-zA-Z0-9_<>]+\\)()"
-  "Regular Expression matching stacktrace information.
+  "> \\([^\"(<]+\\)(\\([0-9]+\\))\\([?a-zA-Z0-9_<>]+\\)()"
+  "Regular expression matching stacktrace information.

 Used to extract the current line and module being inspected."
   :type 'string
   :group 'python
 Used to extract the current line and module being inspected."
   :type 'string
   :group 'python


@@ -2310,7 +2993,9 @@ Never set this variable directly, use
   "Set the buffer for FILE-NAME as the tracked buffer.
 Internally it uses the `python-pdbtrack-tracked-buffer' variable.
 Returns the tracked buffer."
   "Set the buffer for FILE-NAME as the tracked buffer.
 Internally it uses the `python-pdbtrack-tracked-buffer' variable.
 Returns the tracked buffer."

-  (let ((file-buffer (get-file-buffer file-name)))
+  (let ((file-buffer (get-file-buffer
+                      (concat (file-remote-p default-directory)
+                              file-name))))

     (if file-buffer
         (setq python-pdbtrack-tracked-buffer file-buffer)
       (setq file-buffer (find-file-noselect file-name))
     (if file-buffer
         (setq python-pdbtrack-tracked-buffer file-buffer)
       (setq file-buffer (find-file-noselect file-name))


@@ -2374,18 +3059,19 @@ Argument OUTPUT is a string with the output from the comint process."
 \f
 ;;; Symbol completion
 
 \f
 ;;; Symbol completion
 

-(defun python-completion-complete-at-point ()
-  "Complete current symbol at point.
-For this to work the best as possible you should call
+(defun python-completion-at-point ()
+  "Function for `completion-at-point-functions' in `python-mode'.
+For this to work as best as possible you should call

 `python-shell-send-buffer' from time to time so context in
 `python-shell-send-buffer' from time to time so context in

-inferior python process is updated properly."
+inferior Python process is updated properly."

   (let ((process (python-shell-get-process)))
   (let ((process (python-shell-get-process)))

-    (if (not process)
-        (error "Completion needs an inferior Python process running")
-      (python-shell-completion-complete-at-point process))))
+    (when process
+      (python-shell-completion-at-point process))))

 
 

-(add-to-list 'debug-ignored-errors
-             "^Completion needs an inferior Python process running.")
+(define-obsolete-function-alias
+  'python-completion-complete-at-point
+  'python-completion-at-point
+  "25.1")

 
 \f
 ;;; Fill paragraph
 
 \f
 ;;; Fill paragraph


@@ -2423,12 +3109,12 @@ fill parens."
 This affects `python-fill-string' behavior with regards to
 triple quotes positioning.
 
 This affects `python-fill-string' behavior with regards to
 triple quotes positioning.
 

-Possible values are DJANGO, ONETWO, PEP-257, PEP-257-NN,
-SYMMETRIC, and NIL.  A value of NIL won't care about quotes
+Possible values are `django', `onetwo', `pep-257', `pep-257-nn',
+`symmetric', and nil.  A value of nil won't care about quotes

 position and will treat docstrings a normal string, any other
 value may result in one of the following docstring styles:
 
 position and will treat docstrings a normal string, any other
 value may result in one of the following docstring styles:
 

-DJANGO:
+`django':

 
     \"\"\"
     Process foo, return bar.
 
     \"\"\"
     Process foo, return bar.


@@ -2440,7 +3126,7 @@ DJANGO:
     If processing fails throw ProcessingError.
     \"\"\"
 
     If processing fails throw ProcessingError.
     \"\"\"
 

-ONETWO:
+`onetwo':

 
     \"\"\"Process foo, return bar.\"\"\"
 
 
     \"\"\"Process foo, return bar.\"\"\"
 


@@ -2451,7 +3137,7 @@ ONETWO:
 
     \"\"\"
 
 
     \"\"\"
 

-PEP-257:
+`pep-257':

 
     \"\"\"Process foo, return bar.\"\"\"
 
 
     \"\"\"Process foo, return bar.\"\"\"
 


@@ -2461,7 +3147,7 @@ PEP-257:
 
     \"\"\"
 
 
     \"\"\"
 

-PEP-257-NN:
+`pep-257-nn':

 
     \"\"\"Process foo, return bar.\"\"\"
 
 
     \"\"\"Process foo, return bar.\"\"\"
 


@@ -2470,7 +3156,7 @@ PEP-257-NN:
     If processing fails throw ProcessingError.
     \"\"\"
 
     If processing fails throw ProcessingError.
     \"\"\"
 

-SYMMETRIC:
+`symmetric':

 
     \"\"\"Process foo, return bar.\"\"\"
 
 
     \"\"\"Process foo, return bar.\"\"\"
 


@@ -2528,8 +3214,7 @@ JUSTIFY should be used (if applicable) as in `fill-paragraph'."
 (defun python-fill-string (&optional justify)
   "String fill function for `python-fill-paragraph'.
 JUSTIFY should be used (if applicable) as in `fill-paragraph'."
 (defun python-fill-string (&optional justify)
   "String fill function for `python-fill-paragraph'.
 JUSTIFY should be used (if applicable) as in `fill-paragraph'."

-  (let* ((marker (point-marker))
-         (str-start-pos
+  (let* ((str-start-pos

           (set-marker
            (make-marker)
            (or (python-syntax-context 'string)
           (set-marker
            (make-marker)
            (or (python-syntax-context 'string)


@@ -2595,7 +3280,7 @@ JUSTIFY should be used (if applicable) as in `fill-paragraph'."
              ;; Again indent only if a newline is added.
              (indent-according-to-mode))))) t)
 
              ;; Again indent only if a newline is added.
              (indent-according-to-mode))))) t)
 

-(defun python-fill-decorator (&optional justify)
+(defun python-fill-decorator (&optional _justify)

   "Decorator fill function for `python-fill-paragraph'.
 JUSTIFY should be used (if applicable) as in `fill-paragraph'."
   t)
   "Decorator fill function for `python-fill-paragraph'.
 JUSTIFY should be used (if applicable) as in `fill-paragraph'."
   t)


@@ -2606,17 +3291,17 @@ JUSTIFY should be used (if applicable) as in `fill-paragraph'."
   (save-restriction
     (narrow-to-region (progn
                         (while (python-syntax-context 'paren)
   (save-restriction
     (narrow-to-region (progn
                         (while (python-syntax-context 'paren)

-                          (goto-char (1- (point-marker))))
-                        (point-marker)
+                          (goto-char (1- (point))))

                         (line-beginning-position))
                       (progn
                         (when (not (python-syntax-context 'paren))
                           (end-of-line)
                           (when (not (python-syntax-context 'paren))
                             (skip-syntax-backward "^)")))
                         (line-beginning-position))
                       (progn
                         (when (not (python-syntax-context 'paren))
                           (end-of-line)
                           (when (not (python-syntax-context 'paren))
                             (skip-syntax-backward "^)")))

-                        (while (python-syntax-context 'paren)
-                          (goto-char (1+ (point-marker))))
-                        (point-marker)))
+                        (while (and (python-syntax-context 'paren)
+                                    (not (eobp)))
+                          (goto-char (1+ (point))))
+                        (point)))

     (let ((paragraph-start "\f\\|[ \t]*$")
           (paragraph-separate ",")
           (fill-paragraph-function))
     (let ((paragraph-start "\f\\|[ \t]*$")
           (paragraph-separate ",")
           (fill-paragraph-function))


@@ -2625,7 +3310,8 @@ JUSTIFY should be used (if applicable) as in `fill-paragraph'."
     (while (not (eobp))
       (forward-line 1)
       (python-indent-line)
     (while (not (eobp))
       (forward-line 1)
       (python-indent-line)

-      (goto-char (line-end-position)))) t)
+      (goto-char (line-end-position))))
+  t)

 
 \f
 ;;; Skeletons
 
 \f
 ;;; Skeletons


@@ -2644,8 +3330,8 @@ the if condition."
 (defvar python-skeleton-available '()
   "Internal list of available skeletons.")
 
 (defvar python-skeleton-available '()
   "Internal list of available skeletons.")
 

-(define-abbrev-table 'python-mode-abbrev-table ()
-  "Abbrev table for Python mode."
+(define-abbrev-table 'python-mode-skeleton-abbrev-table ()
+  "Abbrev table for Python mode skeletons."

   :case-fixed t
   ;; Allow / inside abbrevs.
   :regexp "\\(?:^\\|[^/]\\)\\<\\([[:word:]/]+\\)\\W*"
   :case-fixed t
   ;; Allow / inside abbrevs.
   :regexp "\\(?:^\\|[^/]\\)\\<\\([[:word:]/]+\\)\\W*"


@@ -2658,13 +3344,13 @@ the if condition."
 (defmacro python-skeleton-define (name doc &rest skel)
   "Define a `python-mode' skeleton using NAME DOC and SKEL.
 The skeleton will be bound to python-skeleton-NAME and will
 (defmacro python-skeleton-define (name doc &rest skel)
   "Define a `python-mode' skeleton using NAME DOC and SKEL.
 The skeleton will be bound to python-skeleton-NAME and will

-be added to `python-mode-abbrev-table'."
+be added to `python-mode-skeleton-abbrev-table'."

   (declare (indent 2))
   (let* ((name (symbol-name name))
          (function-name (intern (concat "python-skeleton-" name))))
     `(progn
   (declare (indent 2))
   (let* ((name (symbol-name name))
          (function-name (intern (concat "python-skeleton-" name))))
     `(progn

-       (define-abbrev python-mode-abbrev-table ,name "" ',function-name
-         :system t)
+       (define-abbrev python-mode-skeleton-abbrev-table
+         ,name "" ',function-name :system t)

        (setq python-skeleton-available
              (cons ',function-name python-skeleton-available))
        (define-skeleton ,function-name
        (setq python-skeleton-available
              (cons ',function-name python-skeleton-available))
        (define-skeleton ,function-name


@@ -2672,6 +3358,10 @@ be added to `python-mode-abbrev-table'."
               (format "Insert %s statement." name))
          ,@skel))))
 
               (format "Insert %s statement." name))
          ,@skel))))
 

+(define-abbrev-table 'python-mode-abbrev-table ()
+  "Abbrev table for Python mode."
+  :parents (list python-mode-skeleton-abbrev-table))
+

 (defmacro python-define-auxiliary-skeleton (name doc &optional &rest skel)
   "Define a `python-mode' auxiliary skeleton using NAME DOC and SKEL.
 The skeleton will be bound to python-skeleton-NAME."
 (defmacro python-define-auxiliary-skeleton (name doc &optional &rest skel)
   "Define a `python-mode' auxiliary skeleton using NAME DOC and SKEL.
 The skeleton will be bound to python-skeleton-NAME."


@@ -2720,6 +3410,12 @@ The skeleton will be bound to python-skeleton-NAME."
   > _ \n
   '(python-skeleton--else) | ^)
 
   > _ \n
   '(python-skeleton--else) | ^)
 

+(python-skeleton-define import nil
+  "Import from module: "
+  "from " str & " " | -5
+  "import "
+  ("Identifier: " str ", ") -2 \n _)
+

 (python-skeleton-define try nil
   nil
   "try:" \n
 (python-skeleton-define try nil
   nil
   "try:" \n


@@ -2746,15 +3442,14 @@ The skeleton will be bound to python-skeleton-NAME."
   "class " str "(" ("Inheritance, %s: "
                     (unless (equal ?\( (char-before)) ", ")
                     str)
   "class " str "(" ("Inheritance, %s: "
                     (unless (equal ?\( (char-before)) ", ")
                     str)

-  & ")" | -2
+  & ")" | -1

   ":" \n
   "\"\"\"" - "\"\"\"" \n
   > _ \n)
 
 (defun python-skeleton-add-menu-items ()
   "Add menu items to Python->Skeletons menu."
   ":" \n
   "\"\"\"" - "\"\"\"" \n
   > _ \n)
 
 (defun python-skeleton-add-menu-items ()
   "Add menu items to Python->Skeletons menu."

-  (let ((skeletons (sort python-skeleton-available 'string<))
-        (items))
+  (let ((skeletons (sort python-skeleton-available 'string<)))

     (dolist (skeleton skeletons)
       (easy-menu-add-item
        nil '("Python" "Skeletons")
     (dolist (skeleton skeletons)
       (easy-menu-add-item
        nil '("Python" "Skeletons")


@@ -2798,6 +3493,8 @@ The skeleton will be bound to python-skeleton-NAME."
         (when module-file
           (substring-no-properties module-file 1 -1))))))
 
         (when module-file
           (substring-no-properties module-file 1 -1))))))
 

+(defvar ffap-alist)
+

 (eval-after-load "ffap"
   '(progn
      (push '(python-mode . python-ffap-module-path) ffap-alist)
 (eval-after-load "ffap"
   '(progn
      (push '(python-mode . python-ffap-module-path) ffap-alist)


@@ -2823,8 +3520,8 @@ The skeleton will be bound to python-skeleton-NAME."
 
 (defun python-check (command)
   "Check a Python file (default current buffer's file).
 
 (defun python-check (command)
   "Check a Python file (default current buffer's file).

-Runs COMMAND, a shell command, as if by `compile'.  See
-`python-check-command' for the default."
+Runs COMMAND, a shell command, as if by `compile'.
+See `python-check-command' for the default."

   (interactive
    (list (read-string "Check command: "
                       (or python-check-custom-command
   (interactive
    (list (read-string "Check command: "
                       (or python-check-custom-command


@@ -2840,7 +3537,7 @@ Runs COMMAND, a shell command, as if by `compile'.  See
   (let ((process-environment (python-shell-calculate-process-environment))
         (exec-path (python-shell-calculate-exec-path)))
     (compilation-start command nil
   (let ((process-environment (python-shell-calculate-process-environment))
         (exec-path (python-shell-calculate-exec-path)))
     (compilation-start command nil

-                       (lambda (mode-name)
+                       (lambda (_modename)

                          (format python-check-buffer-name command)))))
 
 \f
                          (format python-check-buffer-name command)))))
 
 \f


@@ -2889,13 +3586,11 @@ Runs COMMAND, a shell command, as if by `compile'.  See
 
 (defun python-eldoc--get-doc-at-point (&optional force-input force-process)
   "Internal implementation to get documentation at point.
 
 (defun python-eldoc--get-doc-at-point (&optional force-input force-process)
   "Internal implementation to get documentation at point.

-If not FORCE-INPUT is passed then what
-`python-info-current-symbol' returns will be used.  If not
-FORCE-PROCESS is passed what `python-shell-get-process' returns
-is used."
+If not FORCE-INPUT is passed then what `python-info-current-symbol'
+returns will be used.  If not FORCE-PROCESS is passed what
+`python-shell-get-process' returns is used."

   (let ((process (or force-process (python-shell-get-process))))
   (let ((process (or force-process (python-shell-get-process))))

-    (if (not process)
-        (error "Eldoc needs an inferior Python process running")
+    (when process

       (let ((input (or force-input
                        (python-info-current-symbol t))))
         (and input
       (let ((input (or force-input
                        (python-info-current-symbol t))))
         (and input


@@ -2905,9 +3600,9 @@ is used."
 
 (defun python-eldoc-function ()
   "`eldoc-documentation-function' for Python.
 
 (defun python-eldoc-function ()
   "`eldoc-documentation-function' for Python.

-For this to work the best as possible you should call
+For this to work as best as possible you should call

 `python-shell-send-buffer' from time to time so context in
 `python-shell-send-buffer' from time to time so context in

-inferior python process is updated properly."
+inferior Python process is updated properly."

   (python-eldoc--get-doc-at-point))
 
 (defun python-eldoc-at-point (symbol)
   (python-eldoc--get-doc-at-point))
 
 (defun python-eldoc-at-point (symbol)


@@ -2922,21 +3617,152 @@ Interactively, prompt for symbol."
                         nil nil symbol))))
   (message (python-eldoc--get-doc-at-point symbol)))
 
                         nil nil symbol))))
   (message (python-eldoc--get-doc-at-point symbol)))
 

-(add-to-list 'debug-ignored-errors
-             "^Eldoc needs an inferior Python process running.")
-

 \f
 ;;; Imenu
 
 \f
 ;;; Imenu
 

-(defun python-imenu-prev-index-position ()
-  "Python mode's `imenu-prev-index-position-function'."
-  (let ((found))
-    (while (and (setq found
-                      (re-search-backward python-nav-beginning-of-defun-regexp nil t))
-                (not (python-info-looking-at-beginning-of-defun))))
-    (and found
-         (python-info-looking-at-beginning-of-defun)
-         (python-info-current-defun))))
+(defvar python-imenu-format-item-label-function
+  'python-imenu-format-item-label
+  "Imenu function used to format an item label.
+It must be a function with two arguments: TYPE and NAME.")
+
+(defvar python-imenu-format-parent-item-label-function
+  'python-imenu-format-parent-item-label
+  "Imenu function used to format a parent item label.
+It must be a function with two arguments: TYPE and NAME.")
+
+(defvar python-imenu-format-parent-item-jump-label-function
+  'python-imenu-format-parent-item-jump-label
+  "Imenu function used to format a parent jump item label.
+It must be a function with two arguments: TYPE and NAME.")
+
+(defun python-imenu-format-item-label (type name)
+  "Return Imenu label for single node using TYPE and NAME."
+  (format "%s (%s)" name type))
+
+(defun python-imenu-format-parent-item-label (type name)
+  "Return Imenu label for parent node using TYPE and NAME."
+  (format "%s..." (python-imenu-format-item-label type name)))
+
+(defun python-imenu-format-parent-item-jump-label (type _name)
+  "Return Imenu label for parent node jump using TYPE and NAME."
+  (if (string= type "class")
+      "*class definition*"
+    "*function definition*"))
+
+(defun python-imenu--put-parent (type name pos tree)
+  "Add the parent with TYPE, NAME and POS to TREE."
+  (let ((label
+         (funcall python-imenu-format-item-label-function type name))
+        (jump-label
+         (funcall python-imenu-format-parent-item-jump-label-function type name)))
+    (if (not tree)
+        (cons label pos)
+      (cons label (cons (cons jump-label pos) tree)))))
+
+(defun python-imenu--build-tree (&optional min-indent prev-indent tree)
+  "Recursively build the tree of nested definitions of a node.
+Arguments MIN-INDENT, PREV-INDENT and TREE are internal and should
+not be passed explicitly unless you know what you are doing."
+  (setq min-indent (or min-indent 0)
+        prev-indent (or prev-indent python-indent-offset))
+  (let* ((pos (python-nav-backward-defun))
+         (type)
+         (name (when (and pos (looking-at python-nav-beginning-of-defun-regexp))
+                 (let ((split (split-string (match-string-no-properties 0))))
+                   (setq type (car split))
+                   (cadr split))))
+         (label (when name
+                  (funcall python-imenu-format-item-label-function type name)))
+         (indent (current-indentation))
+         (children-indent-limit (+ python-indent-offset min-indent)))
+    (cond ((not pos)
+           ;; Nothing found, probably near to bobp.
+           nil)
+          ((<= indent min-indent)
+           ;; The current indentation points that this is a parent
+           ;; node, add it to the tree and stop recursing.
+           (python-imenu--put-parent type name pos tree))
+          (t
+           (python-imenu--build-tree
+            min-indent
+            indent
+            (if (<= indent children-indent-limit)
+                ;; This lies within the children indent offset range,
+                ;; so it's a normal child of its parent (i.e., not
+                ;; a child of a child).
+                (cons (cons label pos) tree)
+              ;; Oh no, a child of a child?!  Fear not, we
+              ;; know how to roll.  We recursively parse these by
+              ;; swapping prev-indent and min-indent plus adding this
+              ;; newly found item to a fresh subtree.  This works, I
+              ;; promise.
+              (cons
+               (python-imenu--build-tree
+                prev-indent indent (list (cons label pos)))
+               tree)))))))
+
+(defun python-imenu-create-index ()
+  "Return tree Imenu alist for the current Python buffer.
+Change `python-imenu-format-item-label-function',
+`python-imenu-format-parent-item-label-function',
+`python-imenu-format-parent-item-jump-label-function' to
+customize how labels are formatted."
+  (goto-char (point-max))
+  (let ((index)
+        (tree))
+    (while (setq tree (python-imenu--build-tree))
+      (setq index (cons tree index)))
+    index))
+
+(defun python-imenu-create-flat-index (&optional alist prefix)
+  "Return flat outline of the current Python buffer for Imenu.
+Optional argument ALIST is the tree to be flattened; when nil
+`python-imenu-build-index' is used with
+`python-imenu-format-parent-item-jump-label-function'
+`python-imenu-format-parent-item-label-function'
+`python-imenu-format-item-label-function' set to
+  (lambda (type name) name)
+Optional argument PREFIX is used in recursive calls and should
+not be passed explicitly.
+
+Converts this:
+
+    ((\"Foo\" . 103)
+     (\"Bar\" . 138)
+     (\"decorator\"
+      (\"decorator\" . 173)
+      (\"wrap\"
+       (\"wrap\" . 353)
+       (\"wrapped_f\" . 393))))
+
+To this:
+
+    ((\"Foo\" . 103)
+     (\"Bar\" . 138)
+     (\"decorator\" . 173)
+     (\"decorator.wrap\" . 353)
+     (\"decorator.wrapped_f\" . 393))"
+  ;; Inspired by imenu--flatten-index-alist removed in revno 21853.
+  (apply
+   'nconc
+   (mapcar
+    (lambda (item)
+      (let ((name (if prefix
+                      (concat prefix "." (car item))
+                    (car item)))
+            (pos (cdr item)))
+        (cond ((or (numberp pos) (markerp pos))
+               (list (cons name pos)))
+              ((listp pos)
+               (cons
+                (cons name (cdar pos))
+                (python-imenu-create-flat-index (cddr item) name))))))
+    (or alist
+        (let* ((fn (lambda (_type name) name))
+               (python-imenu-format-item-label-function fn)
+              (python-imenu-format-parent-item-label-function fn)
+              (python-imenu-format-parent-item-jump-label-function fn))
+          (python-imenu-create-index))))))

 
 \f
 ;;; Misc helpers
 
 \f
 ;;; Misc helpers


@@ -2944,9 +3770,8 @@ Interactively, prompt for symbol."
 (defun python-info-current-defun (&optional include-type)
   "Return name of surrounding function with Python compatible dotty syntax.
 Optional argument INCLUDE-TYPE indicates to include the type of the defun.
 (defun python-info-current-defun (&optional include-type)
   "Return name of surrounding function with Python compatible dotty syntax.
 Optional argument INCLUDE-TYPE indicates to include the type of the defun.

-This function is compatible to be used as
-`add-log-current-defun-function' since it returns nil if point is
-not inside a defun."
+This function can be used as the value of `add-log-current-defun-function'
+since it returns nil if point is not inside a defun."

   (save-restriction
     (widen)
     (save-excursion
   (save-restriction
     (widen)
     (save-excursion


@@ -3065,49 +3890,88 @@ parent defun name."
   (and (python-info-end-of-statement-p)
        (python-info-statement-ends-block-p)))
 
   (and (python-info-end-of-statement-p)
        (python-info-statement-ends-block-p)))
 

-(defun python-info-closing-block ()
-  "Return the point of the block the current line closes."
-  (let ((closing-word (save-excursion
-                        (back-to-indentation)
-                        (current-word)))
-        (indentation (current-indentation)))
-    (when (member closing-word python-indent-dedenters)
+(define-obsolete-function-alias
+  'python-info-closing-block
+  'python-info-dedenter-opening-block-position "24.4")
+
+(defun python-info-dedenter-opening-block-position ()
+  "Return the point of the closest block the current line closes.
+Returns nil if point is not on a dedenter statement or no opening
+block can be detected.  The latter case meaning current file is
+likely an invalid python file."
+  (let ((positions (python-info-dedenter-opening-block-positions))
+        (indentation (current-indentation))
+        (position))
+    (while (and (not position)
+                positions)

       (save-excursion
       (save-excursion

-        (forward-line -1)
-        (while (and (> (current-indentation) indentation)
-                    (not (bobp))
-                    (not (back-to-indentation))
-                    (forward-line -1)))
-        (back-to-indentation)
-        (cond
-         ((not (equal indentation (current-indentation))) nil)
-         ((string= closing-word "elif")
-          (when (member (current-word) '("if" "elif"))
-            (point-marker)))
-         ((string= closing-word "else")
-          (when (member (current-word) '("if" "elif" "except" "for" "while"))
-            (point-marker)))
-         ((string= closing-word "except")
-          (when (member (current-word) '("try"))
-            (point-marker)))
-         ((string= closing-word "finally")
-          (when (member (current-word) '("except" "else"))
-            (point-marker))))))))
-
-(defun python-info-closing-block-message (&optional closing-block-point)
-  "Message the contents of the block the current line closes.
-With optional argument CLOSING-BLOCK-POINT use that instead of
-recalculating it calling `python-info-closing-block'."
-  (let ((point (or closing-block-point (python-info-closing-block))))
+        (goto-char (car positions))
+        (if (<= (current-indentation) indentation)
+            (setq position (car positions))
+          (setq positions (cdr positions)))))
+    position))
+
+(defun python-info-dedenter-opening-block-positions ()
+  "Return points of blocks the current line may close sorted by closer.
+Returns nil if point is not on a dedenter statement or no opening
+block can be detected.  The latter case meaning current file is
+likely an invalid python file."
+  (save-excursion
+    (let ((dedenter-pos (python-info-dedenter-statement-p)))
+      (when dedenter-pos
+        (goto-char dedenter-pos)
+        (let* ((pairs '(("elif" "elif" "if")
+                        ("else" "if" "elif" "except" "for" "while")
+                        ("except" "except" "try")
+                        ("finally" "else" "except" "try")))
+               (dedenter (match-string-no-properties 0))
+               (possible-opening-blocks (cdr (assoc-string dedenter pairs)))
+               (collected-indentations)
+               (opening-blocks))
+          (catch 'exit
+            (while (python-nav--syntactically
+                    (lambda ()
+                      (re-search-backward (python-rx block-start) nil t))
+                    #'<)
+              (let ((indentation (current-indentation)))
+                (when (and (not (memq indentation collected-indentations))
+                           (or (not collected-indentations)
+                               (< indentation (apply #'min collected-indentations))))
+                  (setq collected-indentations
+                        (cons indentation collected-indentations))
+                  (when (member (match-string-no-properties 0)
+                                possible-opening-blocks)
+                    (setq opening-blocks (cons (point) opening-blocks))))
+                (when (zerop indentation)
+                  (throw 'exit nil)))))
+          ;; sort by closer
+          (nreverse opening-blocks))))))
+
+(define-obsolete-function-alias
+  'python-info-closing-block-message
+  'python-info-dedenter-opening-block-message "24.4")
+
+(defun python-info-dedenter-opening-block-message  ()
+  "Message the first line of the block the current statement closes."
+  (let ((point (python-info-dedenter-opening-block-position)))

     (when point
       (save-restriction
         (widen)
         (message "Closes %s" (save-excursion
                                (goto-char point)
     (when point
       (save-restriction
         (widen)
         (message "Closes %s" (save-excursion
                                (goto-char point)

-                               (back-to-indentation)

                                (buffer-substring
                                 (point) (line-end-position))))))))
 
                                (buffer-substring
                                 (point) (line-end-position))))))))
 

+(defun python-info-dedenter-statement-p ()
+  "Return point if current statement is a dedenter.
+Sets `match-data' to the keyword that starts the dedenter
+statement."
+  (save-excursion
+    (python-nav-beginning-of-statement)
+    (when (and (not (python-syntax-context-type))
+               (looking-at (python-rx dedenter)))
+      (point))))
+

 (defun python-info-line-ends-backslash-p (&optional line-number)
   "Return non-nil if current line ends with backslash.
 With optional argument LINE-NUMBER, check that line instead."
 (defun python-info-line-ends-backslash-p (&optional line-number)
   "Return non-nil if current line ends with backslash.
 With optional argument LINE-NUMBER, check that line instead."


@@ -3209,13 +4073,13 @@ operator."
          (looking-at python-nav-beginning-of-defun-regexp))))
 
 (defun python-info-current-line-comment-p ()
          (looking-at python-nav-beginning-of-defun-regexp))))
 
 (defun python-info-current-line-comment-p ()

-  "Check if current line is a comment line."
+  "Return non-nil if current line is a comment line."

   (char-equal
    (or (char-after (+ (line-beginning-position) (current-indentation))) ?_)
    ?#))
 
 (defun python-info-current-line-empty-p ()
   (char-equal
    (or (char-after (+ (line-beginning-position) (current-indentation))) ?_)
    ?#))
 
 (defun python-info-current-line-empty-p ()

-  "Check if current line is empty, ignoring whitespace."
+  "Return non-nil if current line is empty, ignoring whitespace."

   (save-excursion
     (beginning-of-line 1)
     (looking-at
   (save-excursion
     (beginning-of-line 1)
     (looking-at


@@ -3246,6 +4110,18 @@ to \"^python-\"."
                (cdr pair))))
    (buffer-local-variables from-buffer)))
 
                (cdr pair))))
    (buffer-local-variables from-buffer)))
 

+(defvar comint-last-prompt-overlay)     ; Shut up, byte compiler.
+
+(defun python-util-comint-last-prompt ()
+  "Return comint last prompt overlay start and end.
+This is for compatibility with Emacs < 24.4."
+  (cond ((bound-and-true-p comint-last-prompt-overlay)
+         (cons (overlay-start comint-last-prompt-overlay)
+               (overlay-end comint-last-prompt-overlay)))
+        ((bound-and-true-p comint-last-prompt)
+         comint-last-prompt)
+        (t nil)))
+

 (defun python-util-forward-comment (&optional direction)
   "Python mode specific version of `forward-comment'.
 Optional argument DIRECTION defines the direction to move to."
 (defun python-util-forward-comment (&optional direction)
   "Python mode specific version of `forward-comment'.
 Optional argument DIRECTION defines the direction to move to."


@@ -3257,14 +4133,131 @@ Optional argument DIRECTION defines the direction to move to."
       (goto-char comment-start))
     (forward-comment factor)))
 
       (goto-char comment-start))
     (forward-comment factor)))
 

+(defun python-util-list-directories (directory &optional predicate max-depth)
+  "List DIRECTORY subdirs, filtered by PREDICATE and limited by MAX-DEPTH.
+Argument PREDICATE defaults to `identity' and must be a function
+that takes one argument (a full path) and returns non-nil for
+allowed files.  When optional argument MAX-DEPTH is non-nil, stop
+searching when depth is reached, else don't limit."
+  (let* ((dir (expand-file-name directory))
+         (dir-length (length dir))
+         (predicate (or predicate #'identity))
+         (to-scan (list dir))
+         (tally nil))
+    (while to-scan
+      (let ((current-dir (car to-scan)))
+        (when (funcall predicate current-dir)
+          (setq tally (cons current-dir tally)))
+        (setq to-scan (append (cdr to-scan)
+                              (python-util-list-files
+                               current-dir #'file-directory-p)
+                              nil))
+        (when (and max-depth
+                   (<= max-depth
+                       (length (split-string
+                                (substring current-dir dir-length)
+                                "/\\|\\\\" t))))
+          (setq to-scan nil))))
+    (nreverse tally)))
+
+(defun python-util-list-files (dir &optional predicate)
+  "List files in DIR, filtering with PREDICATE.
+Argument PREDICATE defaults to `identity' and must be a function
+that takes one argument (a full path) and returns non-nil for
+allowed files."
+  (let ((dir-name (file-name-as-directory dir)))
+    (apply #'nconc
+           (mapcar (lambda (file-name)
+                     (let ((full-file-name (expand-file-name file-name dir-name)))
+                       (when (and
+                              (not (member file-name '("." "..")))
+                              (funcall (or predicate #'identity) full-file-name))
+                         (list full-file-name))))
+                   (directory-files dir-name)))))
+
+(defun python-util-list-packages (dir &optional max-depth)
+  "List packages in DIR, limited by MAX-DEPTH.
+When optional argument MAX-DEPTH is non-nil, stop searching when
+depth is reached, else don't limit."
+  (let* ((dir (expand-file-name dir))
+         (parent-dir (file-name-directory
+                      (directory-file-name
+                       (file-name-directory
+                        (file-name-as-directory dir)))))
+         (subpath-length (length parent-dir)))
+    (mapcar
+     (lambda (file-name)
+       (replace-regexp-in-string
+        (rx (or ?\\ ?/)) "." (substring file-name subpath-length)))
+     (python-util-list-directories
+      (directory-file-name dir)
+      (lambda (dir)
+        (file-exists-p (expand-file-name "__init__.py" dir)))
+      max-depth))))
+
+(defun python-util-popn (lst n)
+  "Return LST first N elements.
+N should be an integer, when negative its opposite is used.
+When N is bigger than the length of LST, the list is
+returned as is."
+  (let* ((n (min (abs n)))
+         (len (length lst))
+         (acc))
+    (if (> n len)
+        lst
+      (while (< 0 n)
+        (setq acc (cons (car lst) acc)
+              lst (cdr lst)
+              n (1- n)))
+      (reverse acc))))
+
+(defun python-util-text-properties-replace-name
+  (from to &optional start end)
+  "Replace properties named FROM to TO, keeping its value.
+Arguments START and END narrow the buffer region to work on."
+  (save-excursion
+    (goto-char (or start (point-min)))
+    (while (not (eobp))
+      (let ((plist (text-properties-at (point)))
+            (next-change (or (next-property-change (point) (current-buffer))
+                             (or end (point-max)))))
+        (when (plist-get plist from)
+          (let* ((face (plist-get plist from))
+                 (plist (plist-put plist from nil))
+                 (plist (plist-put plist to face)))
+            (set-text-properties (point) next-change plist (current-buffer))))
+        (goto-char next-change)))))
+
+(defun python-util-strip-string (string)
+  "Strip STRING whitespace and newlines from end and beginning."
+  (replace-regexp-in-string
+   (rx (or (: string-start (* (any whitespace ?\r ?\n)))
+           (: (* (any whitespace ?\r ?\n)) string-end)))
+   ""
+   string))
+
+(defun python-util-valid-regexp-p (regexp)
+  "Return non-nil if REGEXP is valid."
+  (ignore-errors (string-match regexp "") t))
+

 \f
 \f

+(defun python-electric-pair-string-delimiter ()
+  (when (and electric-pair-mode
+             (memq last-command-event '(?\" ?\'))
+             (let ((count 0))
+               (while (eq (char-before (- (point) count)) last-command-event)
+                 (cl-incf count))
+               (= count 3))
+             (eq (char-after) last-command-event))
+    (save-excursion (insert (make-string 2 last-command-event)))))
+
+(defvar electric-indent-inhibit)
+

 ;;;###autoload
 (define-derived-mode python-mode prog-mode "Python"
   "Major mode for editing Python files.
 
 ;;;###autoload
 (define-derived-mode python-mode prog-mode "Python"
   "Major mode for editing Python files.
 

-\\{python-mode-map}
-Entry to this mode calls the value of `python-mode-hook'
-if that value is non-nil."
+\\{python-mode-map}"

   (set (make-local-variable 'tab-width) 8)
   (set (make-local-variable 'indent-tabs-mode) nil)
 
   (set (make-local-variable 'tab-width) 8)
   (set (make-local-variable 'indent-tabs-mode) nil)
 


@@ -3286,10 +4279,17 @@ if that value is non-nil."
   (set (make-local-variable 'indent-line-function)
        #'python-indent-line-function)
   (set (make-local-variable 'indent-region-function) #'python-indent-region)
   (set (make-local-variable 'indent-line-function)
        #'python-indent-line-function)
   (set (make-local-variable 'indent-region-function) #'python-indent-region)

+  ;; Because indentation is not redundant, we cannot safely reindent code.
+  (setq-local electric-indent-inhibit t)
+  (setq-local electric-indent-chars (cons ?: electric-indent-chars))
+
+  ;; Add """ ... """ pairing to electric-pair-mode.
+  (add-hook 'post-self-insert-hook
+            #'python-electric-pair-string-delimiter 'append t)

 
   (set (make-local-variable 'paragraph-start) "\\s-*$")
   (set (make-local-variable 'fill-paragraph-function)
 
   (set (make-local-variable 'paragraph-start) "\\s-*$")
   (set (make-local-variable 'fill-paragraph-function)

-       'python-fill-paragraph)
+       #'python-fill-paragraph)

 
   (set (make-local-variable 'beginning-of-defun-function)
        #'python-nav-beginning-of-defun)
 
   (set (make-local-variable 'beginning-of-defun-function)
        #'python-nav-beginning-of-defun)


@@ -3297,16 +4297,13 @@ if that value is non-nil."
        #'python-nav-end-of-defun)
 
   (add-hook 'completion-at-point-functions
        #'python-nav-end-of-defun)
 
   (add-hook 'completion-at-point-functions

-            'python-completion-complete-at-point nil 'local)
+            #'python-completion-at-point nil 'local)

 
   (add-hook 'post-self-insert-hook
 
   (add-hook 'post-self-insert-hook

-            'python-indent-post-self-insert-function nil 'local)
+            #'python-indent-post-self-insert-function 'append 'local)

 
 

-  (set (make-local-variable 'imenu-extract-index-name-function)
-       #'python-info-current-defun)
-
-  (set (make-local-variable 'imenu-prev-index-position-function)
-       #'python-imenu-prev-index-position)
+  (set (make-local-variable 'imenu-create-index-function)
+       #'python-imenu-create-index)

 
   (set (make-local-variable 'add-log-current-defun-function)
        #'python-info-current-defun)
 
   (set (make-local-variable 'add-log-current-defun-function)
        #'python-info-current-defun)


@@ -3324,14 +4321,13 @@ if that value is non-nil."
 
   (add-to-list 'hs-special-modes-alist
                `(python-mode "^\\s-*\\(?:def\\|class\\)\\>" nil "#"
 
   (add-to-list 'hs-special-modes-alist
                `(python-mode "^\\s-*\\(?:def\\|class\\)\\>" nil "#"

-                             ,(lambda (arg)
-                                (python-nav-end-of-defun)) nil))
-
-  (set (make-local-variable 'mode-require-final-newline) t)
+                             ,(lambda (_arg)
+                                (python-nav-end-of-defun))
+                             nil))

 
   (set (make-local-variable 'outline-regexp)
        (python-rx (* space) block-start))
 
   (set (make-local-variable 'outline-regexp)
        (python-rx (* space) block-start))

-  (set (make-local-variable 'outline-heading-end-regexp) ":\\s-*\n")
+  (set (make-local-variable 'outline-heading-end-regexp) ":[^\n]*\n")

   (set (make-local-variable 'outline-level)
        #'(lambda ()
            "`outline-level' function for Python mode."
   (set (make-local-variable 'outline-level)
        #'(lambda ()
            "`outline-level' function for Python mode."