Add support for heterogeneous and string dimensions

examples/fragments_consolidation.py

@@ -77,10 +77,11 @@ def read_array():
         # Read the entire array. To get coord values as well, we use the .query() syntax.
         data = A.query(coords=True)[:, :]
         a_vals = data["a"]
-        coords = data["coords"]
-        for i in range(coords.shape[0]):
-            for j in range(coords.shape[1]):
-                print("Cell {} has data {}".format(str(coords[i, j]), str(a_vals[i, j])))
+        rows = data["rows"]
+        cols = data["cols"]
+        for i in range(rows.shape[0]):
+            for j in range(cols.shape[0]):
+                print("Cell {} has data {}".format(str((rows[i,j], cols[i,j])), str(a_vals[i, j])))
 
 
 # Create and write array only if it does not exist

examples/quickstart_sparse.py

@@ -70,7 +70,7 @@ def read_array():
         # Slice only rows 1, 2 and cols 2, 3, 4.
         data = A[1:3, 2:5]
         a_vals = data["a"]
-        for i, coord in enumerate(data["coords"]):
+        for i, coord in enumerate(zip(data["rows"], data["cols"])):
             print("Cell (%d, %d) has data %d" % (coord[0], coord[1], a_vals[i]))
 
 

examples/reading_dense_layouts.py

@@ -79,7 +79,7 @@ def read_array(order):
         # dense arrays and specify an order other than the default row-major
         data = A.query(attrs=["a"], order=order, coords=True)[1:3, 2:5]
         a_vals = data["a"]
-        coords = data["coords"]
+        coords = np.asarray(list(zip(data["rows"], data["cols"])))
 
         if order != 'G' and a_vals.flags['F_CONTIGUOUS']:
             print("NOTE: The following result array has col-major layout internally")

examples/reading_sparse_layouts.py

@@ -75,10 +75,9 @@ def read_array(order):
         # other than the default row-major
         data = A.query(attrs=["a"], order=order, coords=True)[1:3, 2:5]
         a_vals = data["a"]
-        coords = data["coords"]
 
-        for i in range(coords.shape[0]):
-            print("Cell {} has data {}".format(str(coords[i]), str(a_vals[i])))
+        for i, coord in enumerate(zip(data["rows"], data["cols"])):
+            print("Cell {} has data {}".format(str(coord), str(a_vals[i])))
 
 
 # Check if the array already exists.

examples/writing_sparse_multiple.py

@@ -73,7 +73,7 @@ def read_array():
         # Slice entire array
         data = A[1:5, 1:5]
         a_vals = data["a"]
-        for i, coord in enumerate(data["coords"]):
+        for i, coord in enumerate(zip(data["rows"], data["cols"])):
             print("Cell (%d, %d) has data %d" % (coord[0], coord[1], a_vals[i]))
 
 

misc/azure-ci.yml

@@ -37,6 +37,7 @@ stages:
       displayName: 'Install dependencies'
 
     - script: |
+        # vcvarsall is necessary so that numpy uses the correct compiler
         call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
         python setup.py build_ext --inplace
         python setup.py install

misc/pypi_linux/Dockerfile

@@ -21,8 +21,8 @@ ENV CMAKE /opt/python/cp27-cp27mu/bin/cmake
 
 ###############################################
 # settings (B)
-ENV TILEDB_VERSION 1.7.5
-ENV TILEDB_PY_VERSION 0.5.6
+ENV TILEDB_VERSION 2.0.0
+ENV TILEDB_PY_VERSION 0.6.0
 ###############################################
 # 1) Nothing builds under GCC 4.8 due to default constructor unused-parameter warnings
 # 2) adding -lrt as a work-around for now because python2.7 doesn't link it, but it
@@ -48,7 +48,7 @@ RUN cd /home/tiledb/ && \
   git -C TileDB checkout $TILEDB_VERSION && \
   mkdir build && \
   cd build && \
-  $CMAKE -DTILEDB_S3=ON -DTILEDB_CPP_API=OFF -DTILEDB_HDFS=ON -DTILEDB_TESTS=OFF \
+  $CMAKE -DTILEDB_S3=ON -DTILEDB_CPP_API=ON -DTILEDB_HDFS=ON -DTILEDB_TESTS=OFF \
          -DTILEDB_SERIALIZATION=ON -DTILEDB_FORCE_ALL_DEPS:BOOL=ON \
          -DTILEDB_LOG_OUTPUT_ON_FAILURE:BOOL=ON \
          -DSANITIZER="OFF;-DCOMPILER_SUPPORTS_AVX2:BOOL=FALSE" \

setup.py

@@ -176,7 +176,7 @@ def build_libtiledb(src_dir):
                     "-DTILEDB_S3=ON",
                     "-DTILEDB_HDFS={}".format("ON" if os.name == "posix" else "OFF"),
                     "-DTILEDB_INSTALL_LIBDIR=lib",
-                    "-DTILEDB_CPP_API=OFF",
+                    "-DTILEDB_CPP_API=ON",
                     "-DTILEDB_LOG_OUTPUT_ON_FAILURE=ON",
                     "-DTILEDB_FORCE_ALL_DEPS:BOOL={}".format("ON" if TILEDB_FORCE_ALL_DEPS else "OFF"),
                     "-DTILEDB_SERIALIZATION:BOOL={}".format("ON" if TILEDB_SERIALIZATION else "OFF")
@@ -240,10 +240,12 @@ def find_or_install_libtiledb(setuptools_cmd):
     :param setuptools_cmd: The setuptools command instance.
     """
     tiledb_ext = None
+    core_ext = None
     for ext in setuptools_cmd.distribution.ext_modules:
         if ext.name == "tiledb.libtiledb":
             tiledb_ext = ext
-            break
+        elif ext.name == "tiledb.core":
+            core_ext = ext
 
     # Download, build and locally install TileDB if needed.
     if hasattr(tiledb_ext, 'tiledb_from_source') or not libtiledb_exists(tiledb_ext.library_dirs):
@@ -285,10 +287,13 @@ def do_copy(src, dest):
 
             #
             tiledb_ext.library_dirs += [os.path.join(install_dir, "lib")]
+            core_ext.library_dirs += [os.path.join(install_dir, "lib")]
 
         # Update the TileDB Extension instance with correct paths.
         tiledb_ext.library_dirs += [os.path.join(install_dir, lib_subdir)]
         tiledb_ext.include_dirs += [os.path.join(install_dir, "include")]
+        core_ext.library_dirs += [os.path.join(install_dir, lib_subdir)]
+        core_ext.include_dirs += [os.path.join(install_dir, "include")]
         # Update package_data so the shared object gets installed with the Python module.
         libtiledb_objects = [os.path.join(native_subdir, libname)
                              for libname in libtiledb_library_names()]
@@ -408,6 +413,18 @@ def run(self):
 
         return bdist_egg_cmd
 
+class get_pybind_include(object):
+  """Helper class to determine the pybind11 include path
+  The purpose of this class is to postpone importing pybind11
+  until it is actually installed, so that the ``get_include()``
+  method can be invoked. """
+
+  def __init__(self, user=False):
+    self.user = user
+
+  def __str__(self):
+    import pybind11
+    return pybind11.get_include(self.user)
 
 def cmake_available():
     """
@@ -427,7 +444,8 @@ def setup_requires():
            numpy_required_version,
            'setuptools>=18.0',
            'setuptools_scm>=1.5.4',
-           'wheel>=0.30']
+           'wheel>=0.30',
+           'pybind11']
     # Add cmake requirement if libtiledb is not found and cmake is not available.
     if not libtiledb_exists(LIB_DIRS) and not cmake_available():
         req.append('cmake>=3.11.0')
@@ -438,15 +456,6 @@ def setup_requires():
 if ver < (3,):
     TESTS_REQUIRE.extend(["unittest2", "mock"])
 
-# Global variables
-CXXFLAGS = os.environ.get("CXXFLAGS", "").split()
-if not is_windows():
-  CXXFLAGS.append("-std=c++11")
-  if not TILEDB_DEBUG_BUILD:
-    CXXFLAGS.append("-Wno-deprecated-declarations")
-
-LFLAGS = os.environ.get("LFLAGS", "").split()
-
 # Allow setting (lib) TileDB directory if it is installed on the system
 TILEDB_PATH = os.environ.get("TILEDB_PATH", "")
 
@@ -476,6 +485,18 @@ def setup_requires():
         TILEDBPY_MODULAR = True
         sys.argv.remove(arg)
 
+# Global variables
+CXXFLAGS = os.environ.get("CXXFLAGS", "").split()
+if not is_windows():
+  CXXFLAGS.append("-std=c++11")
+  if not TILEDB_DEBUG_BUILD:
+    CXXFLAGS.append("-Wno-deprecated-declarations")
+  elif TILEDB_DEBUG_BUILD:
+    CXXFLAGS.append("-g")
+    CXXFLAGS.append("-O0")
+    CXXFLAGS.append("-UNDEBUG") # defined by distutils
+LFLAGS = os.environ.get("LFLAGS", "").split()
+
 if TILEDB_PATH != '' and TILEDB_PATH != 'source':
     LIB_DIRS += [os.path.join(TILEDB_PATH, 'lib')]
     if sys.platform.startswith("linux"):
@@ -513,7 +534,19 @@ def setup_requires():
     extra_link_args=LFLAGS,
     extra_compile_args=CXXFLAGS,
     language="c++"
-    )
+    ),
+  Extension(
+    "tiledb.core",
+    ["tiledb/core.cc"],
+    include_dirs = INC_DIRS + [
+        get_pybind_include(),
+        get_pybind_include(user=True)
+    ],
+  language="c++",
+  libraries=LIBS,
+  extra_link_args=LFLAGS,
+  extra_compile_args=CXXFLAGS,
+  )
 ]
 
 if TILEDBPY_MODULAR: