Updates and minor clean-up

- Add tests for string dimension domain - Fix full-range ':' indexing for string dimensions - Fix warnings/deprecations/unused - Fixes for python2
TileDB-Inc · May 4, 2020 · 2aee715 · 2aee715
1 parent 8c59159
commit 2aee715
Show file tree

Hide file tree

Showing 13 changed files with 244 additions and 207 deletions.
diff --git a/cc_tst.py b/cc_tst.py
diff --git a/misc/pypi_linux/Dockerfile b/misc/pypi_linux/Dockerfile
@@ -21,8 +21,8 @@ ENV CMAKE /opt/python/cp27-cp27mu/bin/cmake
 
 ###############################################
 # settings (B)
-ENV TILEDB_VERSION cpp_c_ptr
-ENV TILEDB_PY_VERSION upd_heter_dim2 
+ENV TILEDB_VERSION 2.0.0
+ENV TILEDB_PY_VERSION 0.6.0
 ###############################################
 # 1) Nothing builds under GCC 4.8 due to default constructor unused-parameter warnings
 # 2) adding -lrt as a work-around for now because python2.7 doesn't link it, but it
@@ -44,7 +44,7 @@ ENV CFLAGS -Wno-unused-parameter -lrt -DKJ_USE_EPOLL=0 -D__BIONIC__=1
 RUN cd /home/tiledb/ && \
   source $HOME/perl5/perlbrew/etc/bashrc && \
   perlbrew use perl-5.10.0 && \
-  git clone https://github.com/ihnorton/TileDB && \
+  git clone https://github.com/TileDB-Inc/TileDB && \
   git -C TileDB checkout $TILEDB_VERSION && \
   mkdir build && \
   cd build && \

diff --git a/misc/pypi_linux/build.sh b/misc/pypi_linux/build.sh
@@ -26,43 +26,43 @@ auditwheel repair dist/*.whl
 cd tiledb/tests
 #/opt/python/cp27-cp27mu/bin/python2.7 -m unittest
 
-## build python35 wheel
-#cd /home/tiledb
-#git clone $TILEDB_PY_REPO TileDB-Py35
-#git -C TileDB-Py35 checkout $TILEDB_PY_VERSION
-#
-#cd /home/tiledb/TileDB-Py35
-#/opt/python/cp35-cp35m/bin/python3.5 setup.py build_ext bdist_wheel --tiledb=/usr/local
-#auditwheel repair dist/*.whl
-#/opt/python/cp35-cp35m/bin/python3.5 -m pip install wheelhouse/*.whl
-#cd tiledb/tests
-##/opt/python/cp35-cp35m/bin/python3.5 -m unittest
-
-
-## build python36 wheel
-#cd /home/tiledb
-#git clone $TILEDB_PY_REPO TileDB-Py36
-#git -C TileDB-Py36 checkout $TILEDB_PY_VERSION
-#
-#cd /home/tiledb/TileDB-Py36
-#/opt/python/cp36-cp36m/bin/python3.6 setup.py build_ext bdist_wheel --tiledb=/usr/local
-#auditwheel repair dist/*.whl
-#/opt/python/cp36-cp36m/bin/python3.6 -m pip install wheelhouse/*.whl
-#cd tiledb/tests
-##/opt/python/cp36-cp36m/bin/python3.6 -m unittest
-#
-#
-## build python37 wheel
-#cd /home/tiledb
-#git clone $TILEDB_PY_REPO TileDB-Py37
-#git -C TileDB-Py37 checkout $TILEDB_PY_VERSION
-#
-#cd /home/tiledb/TileDB-Py37
-#/opt/python/cp37-cp37m/bin/python3.7 setup.py build_ext bdist_wheel --tiledb=/usr/local
-#auditwheel repair dist/*.whl
-#/opt/python/cp37-cp37m/bin/python3.7 -m pip install wheelhouse/*.whl
-#cd tiledb/tests
-##/opt/python/cp37-cp37m/bin/python3.7 -m unittest
+# build python35 wheel
+cd /home/tiledb
+git clone $TILEDB_PY_REPO TileDB-Py35
+git -C TileDB-Py35 checkout $TILEDB_PY_VERSION
+
+cd /home/tiledb/TileDB-Py35
+/opt/python/cp35-cp35m/bin/python3.5 setup.py build_ext bdist_wheel --tiledb=/usr/local
+auditwheel repair dist/*.whl
+/opt/python/cp35-cp35m/bin/python3.5 -m pip install wheelhouse/*.whl
+cd tiledb/tests
+#/opt/python/cp35-cp35m/bin/python3.5 -m unittest
+
+
+# build python36 wheel
+cd /home/tiledb
+git clone $TILEDB_PY_REPO TileDB-Py36
+git -C TileDB-Py36 checkout $TILEDB_PY_VERSION
+
+cd /home/tiledb/TileDB-Py36
+/opt/python/cp36-cp36m/bin/python3.6 setup.py build_ext bdist_wheel --tiledb=/usr/local
+auditwheel repair dist/*.whl
+/opt/python/cp36-cp36m/bin/python3.6 -m pip install wheelhouse/*.whl
+cd tiledb/tests
+#/opt/python/cp36-cp36m/bin/python3.6 -m unittest
+
+
+# build python37 wheel
+cd /home/tiledb
+git clone $TILEDB_PY_REPO TileDB-Py37
+git -C TileDB-Py37 checkout $TILEDB_PY_VERSION
+
+cd /home/tiledb/TileDB-Py37
+/opt/python/cp37-cp37m/bin/python3.7 setup.py build_ext bdist_wheel --tiledb=/usr/local
+auditwheel repair dist/*.whl
+/opt/python/cp37-cp37m/bin/python3.7 -m pip install wheelhouse/*.whl
+cd tiledb/tests
+#/opt/python/cp37-cp37m/bin/python3.7 -m unittest
 
 # build python38 wheel
 cd /home/tiledb
@@ -78,7 +78,7 @@ cd tiledb/tests
 
 # copy build products out
 cp /home/tiledb/TileDB-Py27/wheelhouse/* /wheels
-#cp /home/tiledb/TileDB-Py35/wheelhouse/* /wheels
-#cp /home/tiledb/TileDB-Py36/wheelhouse/* /wheels
-#cp /home/tiledb/TileDB-Py37/wheelhouse/* /wheels
+cp /home/tiledb/TileDB-Py35/wheelhouse/* /wheels
+cp /home/tiledb/TileDB-Py36/wheelhouse/* /wheels
+cp /home/tiledb/TileDB-Py37/wheelhouse/* /wheels
 cp /home/tiledb/TileDB-Py38/wheelhouse/* /wheels
diff --git a/tiledb/core.cc b/tiledb/core.cc
@@ -8,7 +8,6 @@
 #include "numpy/arrayobject.h"
 #undef NPY_NO_DEPRECATED_API
 
-
 #include <pybind11/numpy.h>
 #include <pybind11/pybind11.h>
 
@@ -47,14 +46,10 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num);
 
 struct BufferInfo {
 
-  BufferInfo(std::string name,
-             size_t data_nbytes,
-             tiledb_datatype_t data_type,
-             uint32_t cell_val_num,
-             size_t offsets_num,
-             bool isvar = false)
+  BufferInfo(std::string name, size_t data_nbytes, tiledb_datatype_t data_type,
+             uint32_t cell_val_num, size_t offsets_num, bool isvar = false)
 
-             : name(name), type(data_type), cell_val_num(cell_val_num), isvar(isvar) {
+      : name(name), type(data_type), cell_val_num(cell_val_num), isvar(isvar) {
 
     dtype = tiledb_dtype(data_type, cell_val_num);
     elem_nbytes = tiledb_datatype_size(type);
@@ -79,7 +74,7 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num) {
   if (cell_val_num == 1) {
     auto np = py::module::import("numpy");
     auto datetime64 = np.attr("datetime64");
-    switch(type) {
+    switch (type) {
     case TILEDB_INT32:
       return py::dtype("int32");
     case TILEDB_INT64:
@@ -177,8 +172,9 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num) {
     return np_dtype(rec_list);
   }
 
-  TPY_ERROR_LOC("tiledb datatype not understood ('" + tiledb::impl::type_to_str(type)
-                + "', cell_val_num: " + std::to_string(cell_val_num) + ")");
+  TPY_ERROR_LOC("tiledb datatype not understood ('" +
+                tiledb::impl::type_to_str(type) +
+                "', cell_val_num: " + std::to_string(cell_val_num) + ")");
 }
 
 class PyQuery {
@@ -210,7 +206,6 @@ class PyQuery {
 
     tiledb_array_t *c_array_ = (py::capsule)array.attr("__capsule__")();
 
-    /* TBD whether we use the C++ API ... */
     // we never own this pointer, pass own=false
     array_ = std::shared_ptr<tiledb::Array>(new Array(ctx_, c_array_, false),
                                             [](Array *p) {} /* no deleter*/);
@@ -333,13 +328,13 @@ class PyQuery {
       case TILEDB_DATETIME_PS:
       case TILEDB_DATETIME_FS:
       case TILEDB_DATETIME_AS: {
-        using T = int64_t;
         py::dtype dtype = tiledb_dtype(tiledb_type, 1);
         auto dt0 = r0.attr("astype")(dtype);
         auto dt1 = r1.attr("astype")(dtype);
         // TODO, this is suboptimal, should define pybind converter
         auto darray = py::array(py::make_tuple(dt0, dt1));
-        query_->add_range(dim_idx, *(int64_t*)darray.data(0), *(int64_t*)darray.data(1));
+        query_->add_range(dim_idx, *(int64_t *)darray.data(0),
+                          *(int64_t *)darray.data(1));
         break;
       }
       default:
@@ -462,7 +457,7 @@ class PyQuery {
     uint64_t cell_nbytes = tiledb_datatype_size(type);
     if (cell_val_num != TILEDB_VAR_NUM)
       cell_nbytes *= cell_val_num;
-    auto dtype = tiledb_dtype(type,cell_val_num);
+    auto dtype = tiledb_dtype(type, cell_val_num);
 
     uint64_t buf_nbytes = 0;
     uint64_t offsets_num = 0;
@@ -481,25 +476,23 @@ class PyQuery {
       offsets_num = init_buffer_bytes_ / sizeof(uint64_t);
     }
 
-    buffers_.insert(
-        {name, BufferInfo(name, buf_nbytes, type, cell_val_num, offsets_num, var)});
+    buffers_.insert({name, BufferInfo(name, buf_nbytes, type, cell_val_num,
+                                      offsets_num, var)});
   }
 
   void set_buffers() {
     for (auto bp : buffers_) {
       auto name = bp.first;
       const BufferInfo b = bp.second;
-      void* data_ptr =
-          (void*)((char *)b.data.data() + (b.data_vals_read * b.elem_nbytes));
+      void *data_ptr =
+          (void *)((char *)b.data.data() + (b.data_vals_read * b.elem_nbytes));
       uint64_t data_nelem =
           (b.data.size() - (b.data_vals_read * b.elem_nbytes)) / b.elem_nbytes;
 
       if (b.isvar) {
-        uint64_t *offsets_ptr =
-            (uint64_t *)b.offsets.data() + b.offsets_read;
+        uint64_t *offsets_ptr = (uint64_t *)b.offsets.data() + b.offsets_read;
         query_->set_buffer(b.name, offsets_ptr,
-                           b.offsets.size() - b.offsets_read,
-                           data_ptr,
+                           b.offsets.size() - b.offsets_read, data_ptr,
                            data_nelem);
       } else {
         query_->set_buffer(b.name, data_ptr, data_nelem);
@@ -530,7 +523,6 @@ class PyQuery {
       }
     }
 
-    // TODO ignore non-requested attributes
     for (auto attr_pair : schema.attributes()) {
       alloc_buffer(attr_pair.first);
     }
@@ -560,11 +552,10 @@ class PyQuery {
 
       update_read_elem_num();
 
-      // TODO handle linear reallocation
       for (auto bp : buffers_) {
         auto buf = bp.second;
 
-        if ( (buf.data_vals_read * buf.elem_nbytes) < buf.data.nbytes() * 2) {
+        if ((int64_t)(buf.data_vals_read * buf.elem_nbytes) < buf.data.nbytes() * 2) {
           buf.data.resize({buf.data.size() * 2}, false);
 
           if (buf.isvar)
@@ -577,7 +568,6 @@ class PyQuery {
         py::gil_scoped_release release;
         query_->submit();
       }
-
     }
 
     update_read_elem_num();

diff --git a/tiledb/indexing.pyx b/tiledb/indexing.pyx
@@ -137,7 +137,7 @@ cdef dict execute_multi_index(Array array,
 
     cdef:
         Attr attr
-        uint64_t attr_idx
+        Py_ssize_t attr_idx
         bytes battr_name
         unicode attr_name
         np.ndarray attr_array
@@ -315,7 +315,7 @@ cpdef multi_index(Array array, tuple attr_names, tuple ranges,
     #####################
     # we loop over the range tuple left to right and apply
     # (unspecified dimensions are excluded)
-    cdef size_t dim_idx, range_idx
+    cdef Py_ssize_t dim_idx, range_idx
 
     for dim_idx in range(len(ranges)):
         c_dim_idx = <uint32_t>dim_idx

diff --git a/tiledb/libmetadata.pyx b/tiledb/libmetadata.pyx
@@ -17,16 +17,6 @@ cdef extern from "<utility>" namespace "std" nogil:
     cdef unique_ptr[vector[char]] move(unique_ptr[vector[char]])
     #cdef unique_ptr[vector[char]] make_unique(vector[char])
 
-cdef object metadata_value_check(val):
-    if isinstance(val, int):
-        if val > numeric_limits[int64_t].max():
-            raise OverflowError("Overflow integer values not supported!")
-
-    elif (PY_MAJOR_VERSION < 3) and isinstance(val, long):
-        if val > numeric_limits[int64_t].max():
-            raise OverflowError("Overflow integer values not supported!")
-
-
 cdef class PackedBuffer:
     cdef bytes data
     cdef tiledb_datatype_t tdbtype
@@ -274,10 +264,6 @@ cdef object get_metadata(array: Array,
 
     return unpack_metadata_val(value_type, value_num, value)
 
-cdef put_metadata_dict(Array array, kv):
-
-    for k,v in kv.iteritems():
-        put_metadata(array, k, v)
 
 cdef object load_metadata(Array array, unpack=True):
     """

diff --git a/tiledb/libtiledb.pxd b/tiledb/libtiledb.pxd
@@ -849,6 +849,20 @@ cdef extern from "tiledb/tiledb.h":
         void* domain,
         int* isempty) nogil
 
+    int32_t tiledb_array_get_non_empty_domain_from_index(
+        tiledb_ctx_t* ctx,
+        tiledb_array_t* array,
+        uint32_t idx,
+        void* domain,
+        int32_t* is_empty);
+
+    int32_t tiledb_array_get_non_empty_domain_from_name(
+        tiledb_ctx_t* ctx,
+        tiledb_array_t* array,
+        const char* name,
+        void* domain,
+        int32_t* is_empty);
+
     int32_t tiledb_array_get_non_empty_domain_var_size_from_index(
         tiledb_ctx_t* ctx,
         tiledb_array_t* array,