Skip to content

Commit

Permalink
minor clean-ups
Browse files Browse the repository at this point in the history
  • Loading branch information
@ihnorton
ihnorton committed May 4, 2020
1 parent 2410526 commit eef5f01
Show file tree
Hide file tree
Showing 6 changed files with 118 additions and 135 deletions.
64 changes: 0 additions & 64 deletions cc_tst.py
View file

This file was deleted.

6 changes: 3 additions & 3 deletions misc/pypi_linux/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ ENV CMAKE /opt/python/cp27-cp27mu/bin/cmake

###############################################
# settings (B)
ENV TILEDB_VERSION cpp_c_ptr
ENV TILEDB_PY_VERSION upd_heter_dim2
ENV TILEDB_VERSION 2.0.0
ENV TILEDB_PY_VERSION 0.6.0
###############################################
# 1) Nothing builds under GCC 4.8 due to default constructor unused-parameter warnings
# 2) adding -lrt as a work-around for now because python2.7 doesn't link it, but it
Expand All @@ -44,7 +44,7 @@ ENV CFLAGS -Wno-unused-parameter -lrt -DKJ_USE_EPOLL=0 -D__BIONIC__=1
RUN cd /home/tiledb/ && \
source $HOME/perl5/perlbrew/etc/bashrc && \
perlbrew use perl-5.10.0 && \
git clone https://github.com/ihnorton/TileDB && \
git clone https://github.com/TileDB-Inc/TileDB && \
git -C TileDB checkout $TILEDB_VERSION && \
mkdir build && \
cd build && \
Expand Down
80 changes: 40 additions & 40 deletions misc/pypi_linux/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,43 +26,43 @@ auditwheel repair dist/*.whl
cd tiledb/tests
#/opt/python/cp27-cp27mu/bin/python2.7 -m unittest

## build python35 wheel
#cd /home/tiledb
#git clone $TILEDB_PY_REPO TileDB-Py35
#git -C TileDB-Py35 checkout $TILEDB_PY_VERSION
#
#cd /home/tiledb/TileDB-Py35
#/opt/python/cp35-cp35m/bin/python3.5 setup.py build_ext bdist_wheel --tiledb=/usr/local
#auditwheel repair dist/*.whl
#/opt/python/cp35-cp35m/bin/python3.5 -m pip install wheelhouse/*.whl
#cd tiledb/tests
##/opt/python/cp35-cp35m/bin/python3.5 -m unittest


## build python36 wheel
#cd /home/tiledb
#git clone $TILEDB_PY_REPO TileDB-Py36
#git -C TileDB-Py36 checkout $TILEDB_PY_VERSION
#
#cd /home/tiledb/TileDB-Py36
#/opt/python/cp36-cp36m/bin/python3.6 setup.py build_ext bdist_wheel --tiledb=/usr/local
#auditwheel repair dist/*.whl
#/opt/python/cp36-cp36m/bin/python3.6 -m pip install wheelhouse/*.whl
#cd tiledb/tests
##/opt/python/cp36-cp36m/bin/python3.6 -m unittest
#
#
## build python37 wheel
#cd /home/tiledb
#git clone $TILEDB_PY_REPO TileDB-Py37
#git -C TileDB-Py37 checkout $TILEDB_PY_VERSION
#
#cd /home/tiledb/TileDB-Py37
#/opt/python/cp37-cp37m/bin/python3.7 setup.py build_ext bdist_wheel --tiledb=/usr/local
#auditwheel repair dist/*.whl
#/opt/python/cp37-cp37m/bin/python3.7 -m pip install wheelhouse/*.whl
#cd tiledb/tests
##/opt/python/cp37-cp37m/bin/python3.7 -m unittest
# build python35 wheel
cd /home/tiledb
git clone $TILEDB_PY_REPO TileDB-Py35
git -C TileDB-Py35 checkout $TILEDB_PY_VERSION

cd /home/tiledb/TileDB-Py35
/opt/python/cp35-cp35m/bin/python3.5 setup.py build_ext bdist_wheel --tiledb=/usr/local
auditwheel repair dist/*.whl
/opt/python/cp35-cp35m/bin/python3.5 -m pip install wheelhouse/*.whl
cd tiledb/tests
#/opt/python/cp35-cp35m/bin/python3.5 -m unittest


# build python36 wheel
cd /home/tiledb
git clone $TILEDB_PY_REPO TileDB-Py36
git -C TileDB-Py36 checkout $TILEDB_PY_VERSION

cd /home/tiledb/TileDB-Py36
/opt/python/cp36-cp36m/bin/python3.6 setup.py build_ext bdist_wheel --tiledb=/usr/local
auditwheel repair dist/*.whl
/opt/python/cp36-cp36m/bin/python3.6 -m pip install wheelhouse/*.whl
cd tiledb/tests
#/opt/python/cp36-cp36m/bin/python3.6 -m unittest


# build python37 wheel
cd /home/tiledb
git clone $TILEDB_PY_REPO TileDB-Py37
git -C TileDB-Py37 checkout $TILEDB_PY_VERSION

cd /home/tiledb/TileDB-Py37
/opt/python/cp37-cp37m/bin/python3.7 setup.py build_ext bdist_wheel --tiledb=/usr/local
auditwheel repair dist/*.whl
/opt/python/cp37-cp37m/bin/python3.7 -m pip install wheelhouse/*.whl
cd tiledb/tests
#/opt/python/cp37-cp37m/bin/python3.7 -m unittest

# build python38 wheel
cd /home/tiledb
Expand All @@ -78,7 +78,7 @@ cd tiledb/tests

# copy build products out
cp /home/tiledb/TileDB-Py27/wheelhouse/* /wheels
#cp /home/tiledb/TileDB-Py35/wheelhouse/* /wheels
#cp /home/tiledb/TileDB-Py36/wheelhouse/* /wheels
#cp /home/tiledb/TileDB-Py37/wheelhouse/* /wheels
cp /home/tiledb/TileDB-Py35/wheelhouse/* /wheels
cp /home/tiledb/TileDB-Py36/wheelhouse/* /wheels
cp /home/tiledb/TileDB-Py37/wheelhouse/* /wheels
cp /home/tiledb/TileDB-Py38/wheelhouse/* /wheels
43 changes: 17 additions & 26 deletions tiledb/core.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
#include "numpy/arrayobject.h"
#undef NPY_NO_DEPRECATED_API


#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>

Expand Down Expand Up @@ -47,14 +46,10 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num);

struct BufferInfo {

BufferInfo(std::string name,
size_t data_nbytes,
tiledb_datatype_t data_type,
uint32_t cell_val_num,
size_t offsets_num,
bool isvar = false)
BufferInfo(std::string name, size_t data_nbytes, tiledb_datatype_t data_type,
uint32_t cell_val_num, size_t offsets_num, bool isvar = false)

: name(name), type(data_type), cell_val_num(cell_val_num), isvar(isvar) {
: name(name), type(data_type), cell_val_num(cell_val_num), isvar(isvar) {

dtype = tiledb_dtype(data_type, cell_val_num);
elem_nbytes = tiledb_datatype_size(type);
Expand All @@ -79,7 +74,7 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num) {
if (cell_val_num == 1) {
auto np = py::module::import("numpy");
auto datetime64 = np.attr("datetime64");
switch(type) {
switch (type) {
case TILEDB_INT32:
return py::dtype("int32");
case TILEDB_INT64:
Expand Down Expand Up @@ -177,8 +172,9 @@ py::dtype tiledb_dtype(tiledb_datatype_t type, uint32_t cell_val_num) {
return np_dtype(rec_list);
}

TPY_ERROR_LOC("tiledb datatype not understood ('" + tiledb::impl::type_to_str(type)
+ "', cell_val_num: " + std::to_string(cell_val_num) + ")");
TPY_ERROR_LOC("tiledb datatype not understood ('" +
tiledb::impl::type_to_str(type) +
"', cell_val_num: " + std::to_string(cell_val_num) + ")");
}

class PyQuery {
Expand Down Expand Up @@ -210,7 +206,6 @@ class PyQuery {

tiledb_array_t *c_array_ = (py::capsule)array.attr("__capsule__")();

/* TBD whether we use the C++ API ... */
// we never own this pointer, pass own=false
array_ = std::shared_ptr<tiledb::Array>(new Array(ctx_, c_array_, false),
[](Array *p) {} /* no deleter*/);
Expand Down Expand Up @@ -339,7 +334,8 @@ class PyQuery {
auto dt1 = r1.attr("astype")(dtype);
// TODO, this is suboptimal, should define pybind converter
auto darray = py::array(py::make_tuple(dt0, dt1));
query_->add_range(dim_idx, *(int64_t*)darray.data(0), *(int64_t*)darray.data(1));
query_->add_range(dim_idx, *(int64_t *)darray.data(0),
*(int64_t *)darray.data(1));
break;
}
default:
Expand Down Expand Up @@ -462,7 +458,7 @@ class PyQuery {
uint64_t cell_nbytes = tiledb_datatype_size(type);
if (cell_val_num != TILEDB_VAR_NUM)
cell_nbytes *= cell_val_num;
auto dtype = tiledb_dtype(type,cell_val_num);
auto dtype = tiledb_dtype(type, cell_val_num);

uint64_t buf_nbytes = 0;
uint64_t offsets_num = 0;
Expand All @@ -481,25 +477,23 @@ class PyQuery {
offsets_num = init_buffer_bytes_ / sizeof(uint64_t);
}

buffers_.insert(
{name, BufferInfo(name, buf_nbytes, type, cell_val_num, offsets_num, var)});
buffers_.insert({name, BufferInfo(name, buf_nbytes, type, cell_val_num,
offsets_num, var)});
}

void set_buffers() {
for (auto bp : buffers_) {
auto name = bp.first;
const BufferInfo b = bp.second;
void* data_ptr =
(void*)((char *)b.data.data() + (b.data_vals_read * b.elem_nbytes));
void *data_ptr =
(void *)((char *)b.data.data() + (b.data_vals_read * b.elem_nbytes));
uint64_t data_nelem =
(b.data.size() - (b.data_vals_read * b.elem_nbytes)) / b.elem_nbytes;

if (b.isvar) {
uint64_t *offsets_ptr =
(uint64_t *)b.offsets.data() + b.offsets_read;
uint64_t *offsets_ptr = (uint64_t *)b.offsets.data() + b.offsets_read;
query_->set_buffer(b.name, offsets_ptr,
b.offsets.size() - b.offsets_read,
data_ptr,
b.offsets.size() - b.offsets_read, data_ptr,
data_nelem);
} else {
query_->set_buffer(b.name, data_ptr, data_nelem);
Expand Down Expand Up @@ -530,7 +524,6 @@ class PyQuery {
}
}

// TODO ignore non-requested attributes
for (auto attr_pair : schema.attributes()) {
alloc_buffer(attr_pair.first);
}
Expand Down Expand Up @@ -560,11 +553,10 @@ class PyQuery {

update_read_elem_num();

// TODO handle linear reallocation
for (auto bp : buffers_) {
auto buf = bp.second;

if ( (buf.data_vals_read * buf.elem_nbytes) < buf.data.nbytes() * 2) {
if ((buf.data_vals_read * buf.elem_nbytes) < buf.data.nbytes() * 2) {
buf.data.resize({buf.data.size() * 2}, false);

if (buf.isvar)
Expand All @@ -577,7 +569,6 @@ class PyQuery {
py::gil_scoped_release release;
query_->submit();
}

}

update_read_elem_num();
Expand Down
57 changes: 57 additions & 0 deletions tiledb/tests/test_core.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import unittest, os

import tiledb
import numpy as np
from tiledb import TileDBError, core
from tiledb.tests.common import DiskTestCase
from numpy.testing import assert_array_equal

class CoreCCTest(DiskTestCase):

def test_pyquery_basic(self):
ctx = tiledb.default_ctx()
uri = self.path("test_pyquery_basic")
tiledb.from_numpy(uri, np.random.rand(4))

a = tiledb.open(uri)
q = core.PyQuery(ctx, a, ("",), False)

try:
q.test_err("bad foo happened")
except Exception as exc:
assert isinstance(exc, tiledb.TileDBError)
assert exc.message == "bad foo happened"

q.set_ranges([[(0, 3)]])

with self.assertRaises(TileDBError):
q.set_ranges([[(0, 3.0)]])

q.set_ranges([[(0, np.int32(3))]])

with self.assertRaises(TileDBError):
q.set_ranges([[(3, "a")]])

with self.assertRaisesRegex(
TileDBError,
"Failed to cast dim range '\\(1.2344, 5.6789\\)' to dim type UINT64.*$",
):
q.set_ranges([[(1.2344, 5.6789)]])

with self.assertRaisesRegex(
TileDBError,
"Failed to cast dim range '\\('aa', 'bbbb'\\)' to dim type UINT64.*$",
):
q.set_ranges([[("aa", "bbbb")]])

a.close()
a = tiledb.open(uri)
q2 = core.PyQuery(ctx, a, ("",), False)

q2.set_ranges([[(0,3)]])
q2.submit()
res = q2.results()[''][0]
res.dtype = np.double
assert_array_equal(res, a[:])

a.close()
3 changes: 1 addition & 2 deletions tiledb/tests/test_libtiledb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -557,7 +557,7 @@ def test_mixed_string_schema(self):
ctx = tiledb.Ctx()
dims = [
tiledb.Dim(name="dpos", ctx=ctx, domain=(-100.0, 100.0), tile=10, dtype=np.float64),
tiledb.Dim(name="str_index", domain=("a", "bb"), tile=50, dtype=np.bytes_)
tiledb.Dim(name="str_index", domain=(None,None), tile=None, dtype=np.bytes_)
]
dom = tiledb.Domain(*dims)
attrs = [
Expand Down Expand Up @@ -1011,7 +1011,6 @@ def test_open_with_timestamp(self):
schema = tiledb.ArraySchema(ctx=ctx, domain=dom, attrs=(att,))
tiledb.DenseArray.create(self.path("foo"), schema)

print("path is: ", self.path("foo"))
# write
with tiledb.DenseArray(self.path("foo"), mode='w', ctx=ctx) as T:
T[:] = A
Expand Down

0 comments on commit eef5f01

Please sign in to comment.