[python internal] Binary and In-place Operation
Abstract
- Use of a
compound operatorthat will be compiled into INPLACE_* bytecode instruction at the end may result in a bug that is difficult to debug. - Why In-place operation design like this ? to make code more “pythonic” and increase performance in some case (avoid allocating new memory areas. this can be verified by using
sys.getsizeof()to get object’s current memory allocated andid()to checking whether object’s pointer change or not)
What
As python coder, binary operation is the code using arithmetic operator like + , - , * , / , % ..., in-place operation is the code using compound operator like += , -= , /= , *= .... If the variable’s type don’t support in-place operation, it will fallback into binary operation.
In python bytecode, all compound operators will be compiled into INPLACE_* bytecode instructions.
Normally, only some sequence-objects (PySequence) will show a difference between binary operation and in-place operation, they support in-place operation, which sq_inplace_concat or sq_inplace_repeat field not equal 0.
Binary operations remove the top of the stack (TOS) and the second top-most stack item (TOS1) from the stack. They perform the operation, and put the result back on the stack.
In-place operations are like binary operations, in that they remove TOS and TOS1, and push the result back on the stack, but the operation is done in-place when TOS1 supports it, and the resulting TOS may be (but does not have to be) the original TOS1.
Credit: dis - Disassembler for Python bytecode
Python Internal Code
Source Code - PySequence Abstract Methods
// https://github.com/python/cpython/blob/6fd4c8ec7740523bb81191c013118d9d6959bc9d/Objects/abstract.c#L1746
// binary operation
// example: l = [1] + [2]
PyObject *
PySequence_Concat(PyObject *s, PyObject *o)
{
if (s == NULL || o == NULL) {
return null_error();
}
PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence;
if (m && m->sq_concat) {
PyObject *res = m->sq_concat(s, o);
assert(_Py_CheckSlotResult(s, "+", res != NULL));
return res;
}
/* Instances of user classes defining an __add__() method only
have an nb_add slot, not an sq_concat slot. So we fall back
to nb_add if both arguments appear to be sequences. */
if (PySequence_Check(s) && PySequence_Check(o)) {
PyObject *result = BINARY_OP1(s, o, NB_SLOT(nb_add), "+");
if (result != Py_NotImplemented)
return result;
Py_DECREF(result);
}
return type_error("'%.200s' object can't be concatenated", s);
}
// https://github.com/python/cpython/blob/6fd4c8ec7740523bb81191c013118d9d6959bc9d/Objects/abstract.c#L1803
// in-place operation
// example: l = [1], l += [2]
PyObject *
PySequence_InPlaceConcat(PyObject *s, PyObject *o)
{
if (s == NULL || o == NULL) {
return null_error();
}
PySequenceMethods *m = Py_TYPE(s)->tp_as_sequence;
if (m && m->sq_inplace_concat) {
PyObject *res = m->sq_inplace_concat(s, o);
assert(_Py_CheckSlotResult(s, "+=", res != NULL));
return res;
}
if (m && m->sq_concat) {
PyObject *res = m->sq_concat(s, o);
assert(_Py_CheckSlotResult(s, "+", res != NULL));
return res;
}
if (PySequence_Check(s) && PySequence_Check(o)) {
PyObject *result = BINARY_IOP1(s, o, NB_SLOT(nb_inplace_add),
NB_SLOT(nb_add), "+=");
if (result != Py_NotImplemented)
return result;
Py_DECREF(result);
}
return type_error("'%.200s' object can't be concatenated", s);
}
Source Code - List Methods
// https://github.com/python/cpython/blob/2ef73be891eb95064e268341e38e81d008add480/Objects/listobject.c#L2854
static PySequenceMethods list_as_sequence = {
(lenfunc)list_length, /* sq_length */
(binaryfunc)list_concat, /* sq_concat */
(ssizeargfunc)list_repeat, /* sq_repeat */
(ssizeargfunc)list_item, /* sq_item */
0, /* sq_slice */
(ssizeobjargproc)list_ass_item, /* sq_ass_item */
0, /* sq_ass_slice */
(objobjproc)list_contains, /* sq_contains */
(binaryfunc)list_inplace_concat, /* sq_inplace_concat */
(ssizeargfunc)list_inplace_repeat, /* sq_inplace_repeat */
};
// https://github.com/python/cpython/blob/2ef73be891eb95064e268341e38e81d008add480/Objects/listobject.c#L510
// binary operation
static PyObject *
list_concat(PyListObject *a, PyObject *bb)
{
Py_ssize_t size;
Py_ssize_t i;
PyObject **src, **dest;
PyListObject *np;
if (!PyList_Check(bb)) {
PyErr_Format(PyExc_TypeError,
"can only concatenate list (not \"%.200s\") to list",
Py_TYPE(bb)->tp_name);
return NULL;
}
#define b ((PyListObject *)bb)
assert((size_t)Py_SIZE(a) + (size_t)Py_SIZE(b) < PY_SSIZE_T_MAX);
size = Py_SIZE(a) + Py_SIZE(b);
if (size == 0) {
return PyList_New(0);
}
np = (PyListObject *) list_new_prealloc(size);
if (np == NULL) {
return NULL;
}
src = a->ob_item;
dest = np->ob_item;
for (i = 0; i < Py_SIZE(a); i++) {
PyObject *v = src[i];
Py_INCREF(v);
dest[i] = v;
}
src = b->ob_item;
dest = np->ob_item + Py_SIZE(a);
for (i = 0; i < Py_SIZE(b); i++) {
PyObject *v = src[i];
Py_INCREF(v);
dest[i] = v;
}
Py_SET_SIZE(np, size);
return (PyObject *)np;
#undef b
}
// https://github.com/python/cpython/blob/2ef73be891eb95064e268341e38e81d008add480/Objects/listobject.c#L990
// in-place operation
static PyObject *
list_inplace_concat(PyListObject *self, PyObject *other)
{
PyObject *result;
result = list_extend(self, other);
if (result == NULL)
return result;
Py_DECREF(result);
Py_INCREF(self);
return (PyObject *)self;
}
Simulation - In-place operation
def BinaryOps(x, y):
# x = x + y
return x + y
def InPlaceOps(x, y):
if type(x) != type(y):
raise TypeError("unsupport type")
if hasattr(x, "extend"):
return BinaryOps(x, y)
f = getattr(x, "extend")
if not isinstance(f, types.BuiltinFunctionType):
return BinaryOps(x, y)
x.extend(y)
return x
Python Bytecode Instructions
Disassembler
>>> def f_binary_add():
... x = y = 1
... x = x + y
...
>>> dis.dis(f_binary_add)
2 0 LOAD_CONST 1 (1)
2 DUP_TOP
4 STORE_FAST 0 (x)
6 STORE_FAST 1 (y)
3 8 LOAD_FAST 0 (x)
10 LOAD_FAST 1 (y)
12 BINARY_ADD
14 STORE_FAST 0 (x)
16 LOAD_CONST 0 (None)
18 RETURN_VALUE
>>>
>>> def f_inplace_add():
... x = y = [1]
... x += y
...
>>> dis.dis(f_inplace_add)
2 0 LOAD_CONST 1 (1)
2 BUILD_LIST 1
4 DUP_TOP
6 STORE_FAST 0 (x)
8 STORE_FAST 1 (y)
3 10 LOAD_FAST 0 (x)
12 LOAD_FAST 1 (y)
14 INPLACE_ADD
16 STORE_FAST 0 (x)
18 LOAD_CONST 0 (None)
20 RETURN_VALUE
>>>
Example
>>> def f_binary_add():
... x = y = [1]
... print(x, y)
... print(id(x), id(y))
... x = x + y
... print(x, y)
... print(id(x), id(y))
...
>>> f_binary_add()
([1], [1])
(58784776L, 58784776L)
([1, 1], [1])
(58784904L, 58784776L)
>>>
>>> def f_inplace_add():
... x = y = [1]
... print(x, y)
... print(id(x), id(y))
... x += y
... print(x, y)
... print(id(x), id(y))
...
>>> f_inplace_add()
([1], [1])
(58784776L, 58784776L)
([1, 1], [1, 1])
(58784776L, 58784776L)
>>>
>>>
>>> # type int not support in-place operation
...
>>> def f_inplace_add():
... x = y = 1
... print(x, y)
... print(id(x), id(y))
... x += y
... print(x, y)
... print(id(x), id(y))
...
>>> f_inplace_add()
(1, 1)
(57040968L, 57040968L)
(2, 1)
(57040944L, 57040968L)
>>>