Merge pull request numpy#16389 from seberg/hardcode-scalar-buffers

ENH: Hardcode buffer handling for simple scalars

Author: mattip

benchmarks/benchmarks/bench_scalar.py

@@ -0,0 +1,33 @@
+from .common import Benchmark, TYPES1
+
+import numpy as np
+
+
+class ScalarMath(Benchmark):
+    # Test scalar math, note that each of these is run repeatedly to offset
+    # the function call overhead to some degree.
+    params = [TYPES1]
+    param_names = ["type"]
+    def setup(self, typename):
+        self.num = np.dtype(typename).type(2)
+
+    def time_addition(self, typename):
+        n = self.num
+        res = n + n + n + n + n + n + n + n + n + n
+
+    def time_addition_pyint(self, typename):
+        n = self.num
+        res = n + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1
+
+    def time_multiplication(self, typename):
+        n = self.num
+        res = n * n * n * n * n * n * n * n * n * n
+
+    def time_power_of_two(self, typename):
+        n = self.num
+        res = n**2, n**2, n**2, n**2, n**2, n**2, n**2, n**2, n**2, n**2
+
+    def time_abs(self, typename):
+        n = self.num
+        res = abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(n))))))))))
+

numpy/core/include/numpy/arrayscalars.h

@@ -140,6 +140,7 @@ typedef struct {
         /* note that the PyObject_HEAD macro lives right here */
         PyUnicodeObject base;
         Py_UCS4 *obval;
+        char *buffer_fmt;
 } PyUnicodeScalarObject;
 
 

numpy/core/src/multiarray/arraytypes.c.src

@@ -928,7 +928,6 @@ VOID_setitem(PyObject *op, void *input, void *vap)
             memset(ip + view.len, 0, itemsize - view.len);
         }
         PyBuffer_Release(&view);
-        _dealloc_cached_buffer_info(op);
     }
     return 0;
 }

numpy/core/src/multiarray/buffer.c

@@ -802,7 +802,7 @@ array_getbuffer(PyObject *obj, Py_buffer *view, int flags)
  * Retrieving buffers for scalars
  */
 int
-gentype_getbuffer(PyObject *self, Py_buffer *view, int flags)
+void_getbuffer(PyObject *self, Py_buffer *view, int flags)
 {
     _buffer_info_t *info = NULL;
     PyArray_Descr *descr = NULL;

numpy/core/src/multiarray/common.c

@@ -299,7 +299,6 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims,
             PyErr_Clear();
             dtype = _descriptor_from_pep3118_format(buffer_view.format);
             PyBuffer_Release(&buffer_view);
-            _dealloc_cached_buffer_info(obj);
             if (dtype) {
                 goto promote_types;
             }
@@ -311,7 +310,6 @@ PyArray_DTypeFromObjectHelper(PyObject *obj, int maxdims,
             dtype = PyArray_DescrNewFromType(NPY_VOID);
             dtype->elsize = buffer_view.itemsize;
             PyBuffer_Release(&buffer_view);
-            _dealloc_cached_buffer_info(obj);
             goto promote_types;
         }
         else {
@@ -626,14 +624,6 @@ _IsWriteable(PyArrayObject *ap)
         return NPY_FALSE;
     }
     PyBuffer_Release(&view);
-    /*
-     * The first call to PyObject_GetBuffer stores a reference to a struct
-     * _buffer_info_t (from buffer.c, with format, ndim, strides and shape) in
-     * a static dictionary, with id(base) as the key. Usually we release it
-     * after the call to PyBuffer_Release, via a call to
-     * _dealloc_cached_buffer_info, but in this case leave it in the cache to
-     * speed up future calls to _IsWriteable.
-     */
     return NPY_TRUE;
 }
 

numpy/core/src/multiarray/conversion_utils.c

@@ -207,7 +207,6 @@ PyArray_BufferConverter(PyObject *obj, PyArray_Chunk *buf)
      * sticks around after the release.
      */
     PyBuffer_Release(&view);
-    _dealloc_cached_buffer_info(obj);
 
     /* Point to the base of the buffer object if present */
     if (PyMemoryView_Check(obj)) {

numpy/core/src/multiarray/ctors.c

@@ -2576,7 +2576,6 @@ PyArray_FromInterface(PyObject *origin)
          * sticks around after the release.
          */
         PyBuffer_Release(&view);
-        _dealloc_cached_buffer_info(base);
 
         /* Get offset number from interface specification */
         attr = _PyDict_GetItemStringWithError(iface, "offset");
@@ -3801,7 +3800,6 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
      * sticks around after the release.
      */
     PyBuffer_Release(&view);
-    _dealloc_cached_buffer_info(buf);
 
     if ((offset < 0) || (offset > ts)) {
         PyErr_Format(PyExc_ValueError,

numpy/core/src/multiarray/descriptor.c

@@ -1808,7 +1808,6 @@ arraydescr_dealloc(PyArray_Descr *self)
         Py_INCREF(self);
         return;
     }
-    _dealloc_cached_buffer_info((PyObject*)self);
     Py_XDECREF(self->typeobj);
     Py_XDECREF(self->names);
     Py_XDECREF(self->fields);

numpy/core/src/multiarray/getset.c

@@ -147,7 +147,6 @@ array_strides_set(PyArrayObject *self, PyObject *obj)
         offset = PyArray_BYTES(self) - (char *)view.buf;
         numbytes = view.len + offset;
         PyBuffer_Release(&view);
-        _dealloc_cached_buffer_info((PyObject*)new);
     }
     else {
         PyErr_Clear();
@@ -376,7 +375,6 @@ array_data_set(PyArrayObject *self, PyObject *op)
      * sticks around after the release.
      */
     PyBuffer_Release(&view);
-    _dealloc_cached_buffer_info(op);
 
     if (!PyArray_ISONESEGMENT(self)) {
         PyErr_SetString(PyExc_AttributeError,

numpy/core/src/multiarray/npy_buffer.h

@@ -10,6 +10,6 @@ NPY_NO_EXPORT PyArray_Descr*
 _descriptor_from_pep3118_format(char const *s);
 
 NPY_NO_EXPORT int
-gentype_getbuffer(PyObject *obj, Py_buffer *view, int flags);
+void_getbuffer(PyObject *obj, Py_buffer *view, int flags);
 
 #endif

numpy/core/src/multiarray/scalartypes.c.src

@@ -85,7 +85,6 @@ gentype_alloc(PyTypeObject *type, Py_ssize_t nitems)
 static void
 gentype_dealloc(PyObject *v)
 {
-    _dealloc_cached_buffer_info(v);
     Py_TYPE(v)->tp_free(v);
 }
 
@@ -1691,7 +1690,6 @@ gentype_reduce(PyObject *self, PyObject *NPY_UNUSED(args))
          * sticks around after the release.
          */
         PyBuffer_Release(&view);
-        _dealloc_cached_buffer_info(self);
     }
     else {
         Py_DECREF(ret);
@@ -2365,9 +2363,167 @@ static PySequenceMethods voidtype_as_sequence = {
 };
 
 
-static PyBufferProcs gentype_as_buffer = {
-    .bf_getbuffer = gentype_getbuffer,
-    /* release buffer not defined (see buffer.c) */
+
+/**begin repeat
+ * #name = bool, byte, short, int, long, longlong, ubyte, ushort, uint, ulong,
+ *         ulonglong, half, float, double, longdouble, cfloat, cdouble,
+ *         clongdouble#
+ * #Name = Bool, Byte, Short, Int, Long, LongLong, UByte, UShort, UInt, ULong,
+ *         ULongLong, Half, Float, Double, LongDouble, CFloat, CDouble,
+ *         CLongDouble#
+ * #NAME = BOOL, BYTE, SHORT, INT, LONG, LONGLONG, UBYTE, USHORT, UINT, ULONG,
+ *         ULONGLONG, HALF, FLOAT, DOUBLE, LONGDOUBLE, CFLOAT, CDOUBLE,
+ *         CLONGDOUBLE#
+ * #fmt = ?, b, h, i, l, q, B, H, I, L, Q, e, f, d, g, Zf, Zd, Zg#
+ */
+
+static int
+@name@_getbuffer(PyObject *self, Py_buffer *view, int flags)
+{
+    if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) {
+        return -1;
+    }
+    Py@Name@ScalarObject *scalar = (Py@Name@ScalarObject *)self;
+
+    static char fmt[3] = "@fmt@";
+
+    view->ndim = 0;
+    view->len = sizeof(scalar->obval);
+    view->itemsize = sizeof(scalar->obval);
+    view->shape = NULL;
+    view->strides = NULL;
+    view->suboffsets = NULL;
+    Py_INCREF(self);
+    view->obj = self;
+    view->buf = &(scalar->obval);
+
+    if ((flags & PyBUF_FORMAT) != PyBUF_FORMAT) {
+        /* It is unnecessary to find the correct format */
+        view->format = NULL;
+        return 0;
+    }
+
+    view->format = fmt;
+
+    return 0;
+}
+
+static PyBufferProcs @name@_arrtype_as_buffer = {
+    .bf_getbuffer = @name@_getbuffer,
+    /* No need to release the buffer */
+};
+
+/**end repeat**/
+
+static int
+unicode_getbuffer(PyObject *self, Py_buffer *view, int flags)
+{
+    if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) {
+        return -1;
+    }
+    PyUnicodeScalarObject *scalar = (PyUnicodeScalarObject *)self;
+    Py_ssize_t length = PyUnicode_GetLength(self);
+
+    view->ndim = 0;
+    view->len = length * 4;
+    view->itemsize = length * 4;
+    view->shape = NULL;
+    view->strides = NULL;
+    view->suboffsets = NULL;
+    Py_INCREF(self);
+    view->obj = self;
+
+    if (scalar->obval == NULL) {
+        /*
+         * Unicode may not have the representation available, `scalar_value`
+         * ensures materialization.
+         */
+        PyArray_Descr *descr = PyArray_DescrFromType(NPY_UNICODE);
+        scalar_value(self, descr);
+        Py_DECREF(descr);
+        if (scalar->obval == NULL) {
+            /* allocating memory failed */
+            Py_SETREF(view->obj, NULL);
+            return -1;
+        }
+    }
+    view->buf = scalar->obval;
+
+    if ((flags & PyBUF_FORMAT) != PyBUF_FORMAT) {
+        /* It is unnecessary to find the correct format */
+        view->format = NULL;
+        return 0;
+    }
+
+    if (scalar->buffer_fmt != NULL) {
+        view->format = scalar->buffer_fmt;
+    }
+    else {
+        scalar->buffer_fmt = PyObject_Malloc(22);
+        if (scalar->buffer_fmt == NULL) {
+            Py_SETREF(view->obj, NULL);
+            return -1;
+        }
+        PyOS_snprintf(scalar->buffer_fmt, 22, "%" NPY_INTP_FMT "w", length);
+        view->format = scalar->buffer_fmt;
+    }
+
+    return 0;
+}
+
+static PyBufferProcs unicode_arrtype_as_buffer = {
+    .bf_getbuffer = unicode_getbuffer,
+    /* No need to release the buffer */
+};
+
+
+/**begin repeat
+ * #name = datetime, timedelta#
+ * #Name = Datetime, Timedelta#
+ */
+
+static int
+@name@_getbuffer(PyObject *self, Py_buffer *view, int flags)
+{
+    if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) {
+        return -1;
+    }
+    Py@Name@ScalarObject *scalar = (Py@Name@ScalarObject *)self;
+
+    view->ndim = 1;
+    view->len = 8;
+    view->itemsize = 1;
+    static Py_ssize_t length = 8;
+    view->shape = &length;
+    view->strides = NULL;
+    view->suboffsets = NULL;
+    Py_INCREF(self);
+    view->obj = self;
+
+    view->buf = &(scalar->obval);
+
+    if ((flags & PyBUF_FORMAT) != PyBUF_FORMAT) {
+        /* It is unnecessary to find the correct format */
+        view->format = NULL;
+        return 0;
+    }
+
+    /* export datetime scalars as bytes (although arrays are not exported) */
+    view->format = "B";
+
+    return 0;
+}
+
+static PyBufferProcs @name@_arrtype_as_buffer = {
+        .bf_getbuffer = @name@_getbuffer,
+        /* No need to release the buffer */
+};
+
+/**end repeat**/
+
+static PyBufferProcs void_arrtype_as_buffer = {
+        .bf_getbuffer = void_getbuffer,  /* defined in buffer.c */
+        /* No need to release the buffer */
 };
 
 
@@ -3584,7 +3740,6 @@ initialize_numeric_types(void)
     init_basetypes();
     PyGenericArrType_Type.tp_dealloc = (destructor)gentype_dealloc;
     PyGenericArrType_Type.tp_as_number = &gentype_as_number;
-    PyGenericArrType_Type.tp_as_buffer = &gentype_as_buffer;
     PyGenericArrType_Type.tp_as_mapping = &gentype_as_mapping;
     PyGenericArrType_Type.tp_flags = BASEFLAGS;
     PyGenericArrType_Type.tp_methods = gentype_methods;
@@ -3668,10 +3823,15 @@ initialize_numeric_types(void)
     Py@NAME@ArrType_Type.tp_new = @name@_arrtype_new;
     Py@NAME@ArrType_Type.tp_richcompare = gentype_richcompare;
 
+#define _IS_@NAME@  /* inherit string buffer */
+#if !defined(_IS_String)
+    Py@NAME@ArrType_Type.tp_as_buffer = &@name@_arrtype_as_buffer;
+#endif
+#undef _IS_@NAME@
+
     /**end repeat**/
 
     PyUnicodeArrType_Type.tp_dealloc = unicode_arrtype_dealloc;
-    PyUnicodeArrType_Type.tp_as_buffer = &gentype_as_buffer;
 
     /**begin repeat
      * #name = bool, byte, short, ubyte, ushort, uint, ulong, ulonglong,

numpy/core/tests/test_scalarinherit.py

@@ -5,7 +5,7 @@
 import pytest
 
 import numpy as np
-from numpy.testing import assert_
+from numpy.testing import assert_, assert_raises
 
 
 class A:
@@ -74,13 +74,21 @@ def test_char_radd(self):
         assert_(s + np_s == b'defabc')
         assert_(u + np_u == u'defabc')
 
+        class MyStr(str, np.generic):
+            # would segfault
+            pass
+
+        with assert_raises(TypeError):
+            # Previously worked, but gave completely wrong result
+            ret = s + MyStr('abc')
 
-        class Mystr(str, np.generic):
+        class MyBytes(bytes, np.generic):
             # would segfault
             pass
 
-        ret = s + Mystr('abc')
-        assert_(type(ret) is type(s))
+        ret = s + MyBytes(b'abc')
+        assert(type(ret) is type(s))
+        assert ret == b"defabc"
 
     def test_char_repeat(self):
         np_s = np.string_('abc')