To: vim_dev@googlegroups.com

Subject: Patch 7.3.220

Fcc: outbox

From: Bram Moolenaar <Bram@moolenaar.net>

Mime-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

------------

Patch 7.3.220

Problem:    Python 3: vim.error is a 'str' instead of an 'Exception' object,

            so 'except' or 'raise' it causes a 'SystemError' exception.

            Buffer objects do not support slice assignment.

            When exchanging text between Vim and Python, multibyte texts become

            gabage or cause Unicode Expceptions, etc.

            'py3file' tries to read in the file as Unicode, sometimes causes

            UnicodeDecodeException

Solution:   Fix the problems. (lilydjwg)

Files:      src/if_py_both.h, src/if_python.c, src/if_python3.c

*** ../mercurial/vim73/src/if_py_both.h	2011-03-22 15:47:18.000000000 +0100

--- src/if_py_both.h	2011-06-18 23:54:25.000000000 +0200

***************

*** 65,74 ****

  OutputWrite(PyObject *self, PyObject *args)

  {

      int len;

!     char *str;

      int error = ((OutputObject *)(self))->error;

!     if (!PyArg_ParseTuple(args, "s#", &str, &len))

  	return NULL;

      Py_BEGIN_ALLOW_THREADS

--- 65,74 ----

  OutputWrite(PyObject *self, PyObject *args)

  {

      int len;

!     char *str = NULL;

      int error = ((OutputObject *)(self))->error;

!     if (!PyArg_ParseTuple(args, "es#", p_enc, &str, &len))

  	return NULL;

      Py_BEGIN_ALLOW_THREADS

***************

*** 76,81 ****

--- 76,82 ----

      writer((writefn)(error ? emsg : msg), (char_u *)str, len);

      Python_Release_Vim();

      Py_END_ALLOW_THREADS

+     PyMem_Free(str);

      Py_INCREF(Py_None);

      return Py_None;

***************

*** 104,113 ****

      for (i = 0; i < n; ++i)

      {

  	PyObject *line = PyList_GetItem(list, i);

! 	char *str;

  	PyInt len;

! 	if (!PyArg_Parse(line, "s#", &str, &len)) {

  	    PyErr_SetString(PyExc_TypeError, _("writelines() requires list of strings"));

  	    Py_DECREF(list);

  	    return NULL;

--- 105,114 ----

      for (i = 0; i < n; ++i)

      {

  	PyObject *line = PyList_GetItem(list, i);

! 	char *str = NULL;

  	PyInt len;

! 	if (!PyArg_Parse(line, "es#", p_enc, &str, &len)) {

  	    PyErr_SetString(PyExc_TypeError, _("writelines() requires list of strings"));

  	    Py_DECREF(list);

  	    return NULL;

***************

*** 118,123 ****

--- 119,125 ----

  	writer((writefn)(error ? emsg : msg), (char_u *)str, len);

  	Python_Release_Vim();

  	Py_END_ALLOW_THREADS

+ 	PyMem_Free(str);

      }

      Py_DECREF(list);

***************

*** 681,686 ****

--- 683,689 ----

  {

      const char *str;

      char *save;

+     PyObject *bytes;

      PyInt len;

      PyInt i;

      char *p;

***************

*** 691,698 ****

  	return NULL;

      }

!     str = PyString_AsString(obj);

!     len = PyString_Size(obj);

      /*

       * Error checking: String must not contain newlines, as we

--- 694,702 ----

  	return NULL;

      }

!     bytes = PyString_AsBytes(obj);  /* for Python 2 this does nothing */

!     str = PyString_AsString(bytes);

!     len = PyString_Size(bytes);

      /*

       * Error checking: String must not contain newlines, as we

***************

*** 731,736 ****

--- 735,741 ----

      }

      save[i] = '\0';

+     PyString_FreeBytes(bytes);  /* Python 2 does nothing here */

      return save;

  }

***************

*** 817,823 ****

      invalidate_botline();

  }

! /* Replace a line in the specified buffer. The line number is

   * in Vim format (1-based). The replacement line is given as

   * a Python string object. The object is checked for validity

   * and correct format. Errors are returned as a value of FAIL.

--- 822,829 ----

      invalidate_botline();

  }

! /*

!  * Replace a line in the specified buffer. The line number is

   * in Vim format (1-based). The replacement line is given as

   * a Python string object. The object is checked for validity

   * and correct format. Errors are returned as a value of FAIL.

***************

*** 908,913 ****

--- 914,1106 ----

      }

  }

+ /* Replace a range of lines in the specified buffer. The line numbers are in

+  * Vim format (1-based). The range is from lo up to, but not including, hi.

+  * The replacement lines are given as a Python list of string objects. The

+  * list is checked for validity and correct format. Errors are returned as a

+  * value of FAIL.  The return value is OK on success.

+  * If OK is returned and len_change is not NULL, *len_change

+  * is set to the change in the buffer length.

+  */

+     static int

+ SetBufferLineList(buf_T *buf, PyInt lo, PyInt hi, PyObject *list, PyInt *len_change)

+ {

+     /* First of all, we check the thpe of the supplied Python object.

+      * There are three cases:

+      *	  1. NULL, or None - this is a deletion.

+      *	  2. A list	   - this is a replacement.

+      *	  3. Anything else - this is an error.

+      */

+     if (list == Py_None || list == NULL)

+     {

+ 	PyInt	i;

+ 	PyInt	n = (int)(hi - lo);

+ 	buf_T	*savebuf = curbuf;

+ 

+ 	PyErr_Clear();

+ 	curbuf = buf;

+ 

+ 	if (u_savedel((linenr_T)lo, (long)n) == FAIL)

+ 	    PyErr_SetVim(_("cannot save undo information"));

+ 	else

+ 	{

+ 	    for (i = 0; i < n; ++i)

+ 	    {

+ 		if (ml_delete((linenr_T)lo, FALSE) == FAIL)

+ 		{

+ 		    PyErr_SetVim(_("cannot delete line"));

+ 		    break;

+ 		}

+ 	    }

+ 	    if (buf == curwin->w_buffer)

+ 		py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)-n);

+ 	    deleted_lines_mark((linenr_T)lo, (long)i);

+ 	}

+ 

+ 	curbuf = savebuf;

+ 

+ 	if (PyErr_Occurred() || VimErrorCheck())

+ 	    return FAIL;

+ 

+ 	if (len_change)

+ 	    *len_change = -n;

+ 

+ 	return OK;

+     }

+     else if (PyList_Check(list))

+     {

+ 	PyInt	i;

+ 	PyInt	new_len = PyList_Size(list);

+ 	PyInt	old_len = hi - lo;

+ 	PyInt	extra = 0;	/* lines added to text, can be negative */

+ 	char	**array;

+ 	buf_T	*savebuf;

+ 

+ 	if (new_len == 0)	/* avoid allocating zero bytes */

+ 	    array = NULL;

+ 	else

+ 	{

+ 	    array = (char **)alloc((unsigned)(new_len * sizeof(char *)));

+ 	    if (array == NULL)

+ 	    {

+ 		PyErr_NoMemory();

+ 		return FAIL;

+ 	    }

+ 	}

+ 

+ 	for (i = 0; i < new_len; ++i)

+ 	{

+ 	    PyObject *line = PyList_GetItem(list, i);

+ 

+ 	    array[i] = StringToLine(line);

+ 	    if (array[i] == NULL)

+ 	    {

+ 		while (i)

+ 		    vim_free(array[--i]);

+ 		vim_free(array);

+ 		return FAIL;

+ 	    }

+ 	}

+ 

+ 	savebuf = curbuf;

+ 

+ 	PyErr_Clear();

+ 	curbuf = buf;

+ 

+ 	if (u_save((linenr_T)(lo-1), (linenr_T)hi) == FAIL)

+ 	    PyErr_SetVim(_("cannot save undo information"));

+ 

+ 	/* If the size of the range is reducing (ie, new_len < old_len) we

+ 	 * need to delete some old_len. We do this at the start, by

+ 	 * repeatedly deleting line "lo".

+ 	 */

+ 	if (!PyErr_Occurred())

+ 	{

+ 	    for (i = 0; i < old_len - new_len; ++i)

+ 		if (ml_delete((linenr_T)lo, FALSE) == FAIL)

+ 		{

+ 		    PyErr_SetVim(_("cannot delete line"));

+ 		    break;

+ 		}

+ 	    extra -= i;

+ 	}

+ 

+ 	/* For as long as possible, replace the existing old_len with the

+ 	 * new old_len. This is a more efficient operation, as it requires

+ 	 * less memory allocation and freeing.

+ 	 */

+ 	if (!PyErr_Occurred())

+ 	{

+ 	    for (i = 0; i < old_len && i < new_len; ++i)

+ 		if (ml_replace((linenr_T)(lo+i), (char_u *)array[i], FALSE)

+ 								      == FAIL)

+ 		{

+ 		    PyErr_SetVim(_("cannot replace line"));

+ 		    break;

+ 		}

+ 	}

+ 	else

+ 	    i = 0;

+ 

+ 	/* Now we may need to insert the remaining new old_len. If we do, we

+ 	 * must free the strings as we finish with them (we can't pass the

+ 	 * responsibility to vim in this case).

+ 	 */

+ 	if (!PyErr_Occurred())

+ 	{

+ 	    while (i < new_len)

+ 	    {

+ 		if (ml_append((linenr_T)(lo + i - 1),

+ 					(char_u *)array[i], 0, FALSE) == FAIL)

+ 		{

+ 		    PyErr_SetVim(_("cannot insert line"));

+ 		    break;

+ 		}

+ 		vim_free(array[i]);

+ 		++i;

+ 		++extra;

+ 	    }

+ 	}

+ 

+ 	/* Free any left-over old_len, as a result of an error */

+ 	while (i < new_len)

+ 	{

+ 	    vim_free(array[i]);

+ 	    ++i;

+ 	}

+ 

+ 	/* Free the array of old_len. All of its contents have now

+ 	 * been dealt with (either freed, or the responsibility passed

+ 	 * to vim.

+ 	 */

+ 	vim_free(array);

+ 

+ 	/* Adjust marks. Invalidate any which lie in the

+ 	 * changed range, and move any in the remainder of the buffer.

+ 	 */

+ 	mark_adjust((linenr_T)lo, (linenr_T)(hi - 1),

+ 						  (long)MAXLNUM, (long)extra);

+ 	changed_lines((linenr_T)lo, 0, (linenr_T)hi, (long)extra);

+ 

+ 	if (buf == curwin->w_buffer)

+ 	    py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)extra);

+ 

+ 	curbuf = savebuf;

+ 

+ 	if (PyErr_Occurred() || VimErrorCheck())

+ 	    return FAIL;

+ 

+ 	if (len_change)

+ 	    *len_change = new_len - old_len;

+ 

+ 	return OK;

+     }

+     else

+     {

+ 	PyErr_BadArgument();

+ 	return FAIL;

+     }

+ }

  /* Insert a number of lines into the specified buffer after the specifed line.

   * The line number is in Vim format (1-based). The lines to be inserted are

***************

*** 1108,1113 ****

--- 1301,1340 ----

  	return -1;

      if (new_end)

+ 	*new_end = end + len_change;

+ 

+     return 0;

+ }

+ 

+     static PyInt

+ RBAsSlice(BufferObject *self, PyInt lo, PyInt hi, PyObject *val, PyInt start, PyInt end, PyInt *new_end)

+ {

+     PyInt size;

+     PyInt len_change;

+ 

+     /* Self must be a valid buffer */

+     if (CheckBuffer(self))

+ 	return -1;

+ 

+     /* Sort out the slice range */

+     size = end - start + 1;

+ 

+     if (lo < 0)

+ 	lo = 0;

+     else if (lo > size)

+ 	lo = size;

+     if (hi < 0)

+ 	hi = 0;

+     if (hi < lo)

+ 	hi = lo;

+     else if (hi > size)

+ 	hi = size;

+ 

+     if (SetBufferLineList(self->buf, lo + start, hi + start,

+ 						    val, &len_change) == FAIL)

+ 	return -1;

+ 

+     if (new_end)

  	*new_end = end + len_change;

      return 0;

*** ../mercurial/vim73/src/if_python.c	2011-03-26 18:32:00.000000000 +0100

--- src/if_python.c	2011-06-19 00:02:15.000000000 +0200

***************

*** 56,61 ****

--- 56,65 ----

  static void init_structs(void);

+ /* No-op conversion functions, use with care! */

+ #define PyString_AsBytes(obj) (obj)

+ #define PyString_FreeBytes(obj)

+ 

  #if !defined(FEAT_PYTHON) && defined(PROTO)

  /* Use this to be able to generate prototypes without python being used. */

  # define PyObject Py_ssize_t

***************

*** 129,134 ****

--- 133,139 ----

   */

  # define PyArg_Parse dll_PyArg_Parse

  # define PyArg_ParseTuple dll_PyArg_ParseTuple

+ # define PyMem_Free dll_PyMem_Free

  # define PyDict_SetItemString dll_PyDict_SetItemString

  # define PyErr_BadArgument dll_PyErr_BadArgument

  # define PyErr_Clear dll_PyErr_Clear

***************

*** 189,194 ****

--- 194,200 ----

   */

  static int(*dll_PyArg_Parse)(PyObject *, char *, ...);

  static int(*dll_PyArg_ParseTuple)(PyObject *, char *, ...);

+ static int(*dll_PyMem_Free)(void *);

  static int(*dll_PyDict_SetItemString)(PyObject *dp, char *key, PyObject *item);

  static int(*dll_PyErr_BadArgument)(void);

  static void(*dll_PyErr_Clear)(void);

***************

*** 271,276 ****

--- 277,283 ----

  {

      {"PyArg_Parse", (PYTHON_PROC*)&dll_PyArg_Parse},

      {"PyArg_ParseTuple", (PYTHON_PROC*)&dll_PyArg_ParseTuple},

+     {"PyMem_Free", (PYTHON_PROC*)&dll_PyMem_Free},

      {"PyDict_SetItemString", (PYTHON_PROC*)&dll_PyDict_SetItemString},

      {"PyErr_BadArgument", (PYTHON_PROC*)&dll_PyErr_BadArgument},

      {"PyErr_Clear", (PYTHON_PROC*)&dll_PyErr_Clear},

***************

*** 833,876 ****

  static PyObject *CurrentGetattr(PyObject *, char *);

  static int CurrentSetattr(PyObject *, char *, PyObject *);

- /* Common routines for buffers and line ranges

-  * -------------------------------------------

-  */

- 

-     static PyInt

- RBAssSlice(BufferObject *self, PyInt lo, PyInt hi, PyObject *val, PyInt start, PyInt end, PyInt *new_end)

- {

-     PyInt size;

-     PyInt len_change;

- 

-     /* Self must be a valid buffer */

-     if (CheckBuffer(self))

- 	return -1;

- 

-     /* Sort out the slice range */

-     size = end - start + 1;

- 

-     if (lo < 0)

- 	lo = 0;

-     else if (lo > size)

- 	lo = size;

-     if (hi < 0)

- 	hi = 0;

-     if (hi < lo)

- 	hi = lo;

-     else if (hi > size)

- 	hi = size;

- 

-     if (SetBufferLineList(self->buf, lo + start, hi + start,

- 						    val, &len_change) == FAIL)

- 	return -1;

- 

-     if (new_end)

- 	*new_end = end + len_change;

- 

-     return 0;

- }

- 

  static PySequenceMethods BufferAsSeq = {

      (PyInquiry)		BufferLength,	    /* sq_length,    len(x)   */

      (binaryfunc)	0, /* BufferConcat, */	     /* sq_concat,    x+y      */

--- 840,845 ----

***************

*** 1038,1044 ****

      static PyInt

  BufferAssSlice(PyObject *self, PyInt lo, PyInt hi, PyObject *val)

  {

!     return RBAssSlice((BufferObject *)(self), lo, hi, val, 1,

  		      (PyInt)((BufferObject *)(self))->buf->b_ml.ml_line_count,

  		      NULL);

  }

--- 1007,1013 ----

      static PyInt

  BufferAssSlice(PyObject *self, PyInt lo, PyInt hi, PyObject *val)

  {

!     return RBAsSlice((BufferObject *)(self), lo, hi, val, 1,

  		      (PyInt)((BufferObject *)(self))->buf->b_ml.ml_line_count,

  		      NULL);

  }

***************

*** 1088,1094 ****

      static PyInt

  RangeAssSlice(PyObject *self, PyInt lo, PyInt hi, PyObject *val)

  {

!     return RBAssSlice(((RangeObject *)(self))->buf, lo, hi, val,

  		      ((RangeObject *)(self))->start,

  		      ((RangeObject *)(self))->end,

  		      &((RangeObject *)(self))->end);

--- 1057,1063 ----

      static PyInt

  RangeAssSlice(PyObject *self, PyInt lo, PyInt hi, PyObject *val)

  {

!     return RBAsSlice(((RangeObject *)(self))->buf, lo, hi, val,

  		      ((RangeObject *)(self))->start,

  		      ((RangeObject *)(self))->end,

  		      &((RangeObject *)(self))->end);

***************

*** 1435,1628 ****

   * 4. Utility functions for handling the interface between Vim and Python.

   */

- /* Replace a range of lines in the specified buffer. The line numbers are in

-  * Vim format (1-based). The range is from lo up to, but not including, hi.

-  * The replacement lines are given as a Python list of string objects. The

-  * list is checked for validity and correct format. Errors are returned as a

-  * value of FAIL.  The return value is OK on success.

-  * If OK is returned and len_change is not NULL, *len_change

-  * is set to the change in the buffer length.

-  */

-     static int

- SetBufferLineList(buf_T *buf, PyInt lo, PyInt hi, PyObject *list, PyInt *len_change)

- {

-     /* First of all, we check the thpe of the supplied Python object.

-      * There are three cases:

-      *	  1. NULL, or None - this is a deletion.

-      *	  2. A list	   - this is a replacement.

-      *	  3. Anything else - this is an error.

-      */

-     if (list == Py_None || list == NULL)

-     {

- 	PyInt	i;

- 	PyInt	n = (int)(hi - lo);

- 	buf_T	*savebuf = curbuf;

- 

- 	PyErr_Clear();

- 	curbuf = buf;

- 

- 	if (u_savedel((linenr_T)lo, (long)n) == FAIL)

- 	    PyErr_SetVim(_("cannot save undo information"));

- 	else

- 	{

- 	    for (i = 0; i < n; ++i)

- 	    {

- 		if (ml_delete((linenr_T)lo, FALSE) == FAIL)

- 		{

- 		    PyErr_SetVim(_("cannot delete line"));

- 		    break;

- 		}

- 	    }

- 	    if (buf == curwin->w_buffer)

- 		py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)-n);

- 	    deleted_lines_mark((linenr_T)lo, (long)i);

- 	}

- 

- 	curbuf = savebuf;

- 

- 	if (PyErr_Occurred() || VimErrorCheck())

- 	    return FAIL;

- 

- 	if (len_change)

- 	    *len_change = -n;

- 

- 	return OK;

-     }

-     else if (PyList_Check(list))

-     {

- 	PyInt	i;

- 	PyInt	new_len = PyList_Size(list);

- 	PyInt	old_len = hi - lo;

- 	PyInt	extra = 0;	/* lines added to text, can be negative */

- 	char	**array;

- 	buf_T	*savebuf;

- 

- 	if (new_len == 0)	/* avoid allocating zero bytes */

- 	    array = NULL;

- 	else

- 	{

- 	    array = (char **)alloc((unsigned)(new_len * sizeof(char *)));

- 	    if (array == NULL)

- 	    {

- 		PyErr_NoMemory();

- 		return FAIL;

- 	    }

- 	}

- 

- 	for (i = 0; i < new_len; ++i)

- 	{

- 	    PyObject *line = PyList_GetItem(list, i);

- 

- 	    array[i] = StringToLine(line);

- 	    if (array[i] == NULL)

- 	    {

- 		while (i)

- 		    vim_free(array[--i]);

- 		vim_free(array);

- 		return FAIL;

- 	    }

- 	}

- 

- 	savebuf = curbuf;

- 

- 	PyErr_Clear();

- 	curbuf = buf;

- 

- 	if (u_save((linenr_T)(lo-1), (linenr_T)hi) == FAIL)

- 	    PyErr_SetVim(_("cannot save undo information"));

- 

- 	/* If the size of the range is reducing (ie, new_len < old_len) we

- 	 * need to delete some old_len. We do this at the start, by

- 	 * repeatedly deleting line "lo".

- 	 */

- 	if (!PyErr_Occurred())

- 	{

- 	    for (i = 0; i < old_len - new_len; ++i)

- 		if (ml_delete((linenr_T)lo, FALSE) == FAIL)

- 		{

- 		    PyErr_SetVim(_("cannot delete line"));

- 		    break;

- 		}

- 	    extra -= i;

- 	}

- 

- 	/* For as long as possible, replace the existing old_len with the

- 	 * new old_len. This is a more efficient operation, as it requires

- 	 * less memory allocation and freeing.

- 	 */

- 	if (!PyErr_Occurred())

- 	{

- 	    for (i = 0; i < old_len && i < new_len; ++i)

- 		if (ml_replace((linenr_T)(lo+i), (char_u *)array[i], FALSE)

- 								      == FAIL)

- 		{

- 		    PyErr_SetVim(_("cannot replace line"));

- 		    break;

- 		}

- 	}

- 	else

- 	    i = 0;

- 

- 	/* Now we may need to insert the remaining new old_len. If we do, we

- 	 * must free the strings as we finish with them (we can't pass the

- 	 * responsibility to vim in this case).

- 	 */

- 	if (!PyErr_Occurred())

- 	{

- 	    while (i < new_len)

- 	    {

- 		if (ml_append((linenr_T)(lo + i - 1),

- 					(char_u *)array[i], 0, FALSE) == FAIL)

- 		{

- 		    PyErr_SetVim(_("cannot insert line"));

- 		    break;

- 		}

- 		vim_free(array[i]);

- 		++i;

- 		++extra;

- 	    }

- 	}

- 

- 	/* Free any left-over old_len, as a result of an error */

- 	while (i < new_len)

- 	{

- 	    vim_free(array[i]);

- 	    ++i;

- 	}

- 

- 	/* Free the array of old_len. All of its contents have now

- 	 * been dealt with (either freed, or the responsibility passed

- 	 * to vim.

- 	 */

- 	vim_free(array);

- 

- 	/* Adjust marks. Invalidate any which lie in the

- 	 * changed range, and move any in the remainder of the buffer.

- 	 */

- 	mark_adjust((linenr_T)lo, (linenr_T)(hi - 1),

- 						  (long)MAXLNUM, (long)extra);

- 	changed_lines((linenr_T)lo, 0, (linenr_T)hi, (long)extra);

- 

- 	if (buf == curwin->w_buffer)

- 	    py_fix_cursor((linenr_T)lo, (linenr_T)hi, (linenr_T)extra);

- 

- 	curbuf = savebuf;

- 

- 	if (PyErr_Occurred() || VimErrorCheck())

- 	    return FAIL;

- 

- 	if (len_change)

- 	    *len_change = new_len - old_len;

- 

- 	return OK;

-     }

-     else

-     {

- 	PyErr_BadArgument();

- 	return FAIL;

-     }

- }

- 

  /* Convert a Vim line into a Python string.

   * All internal newlines are replaced by null characters.

   *

--- 1404,1409 ----

*** ../mercurial/vim73/src/if_python3.c	2011-06-12 21:37:06.000000000 +0200

--- src/if_python3.c	2011-06-19 00:10:42.000000000 +0200

***************

*** 70,77 ****

  #define PyInt Py_ssize_t

  #define PyString_Check(obj) PyUnicode_Check(obj)

! #define PyString_AsString(obj) _PyUnicode_AsString(obj)