diff --git a/HISTORY.md b/HISTORY.md index 0a5dfcfb302..e3ac8c2504f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -15,6 +15,7 @@ * Source built curl only need HTTP support [#1712](https://github.com/TileDB-Inc/TileDB/pull/1712) * AWS SDK version bumped to 1.8.6 [#1718](https://github.com/TileDB-Inc/TileDB/pull/1718) * Split posix permissions into files and folers permissions [#1719](https://github.com/TileDB-Inc/TileDB/pull/1719) +* Support seeking for CURL to allow redirects for posting to REST [#1728](https://github.com/TileDB-Inc/TileDB/pull/1728) ## Deprecations diff --git a/tiledb/sm/buffer/buffer_list.cc b/tiledb/sm/buffer/buffer_list.cc index 123ad8c986d..16a6516f822 100644 --- a/tiledb/sm/buffer/buffer_list.cc +++ b/tiledb/sm/buffer/buffer_list.cc @@ -94,7 +94,10 @@ Status BufferList::read(void* dest, uint64_t nbytes, uint64_t* bytes_read) { // Read from buffer const uint64_t bytes_in_src = src.size() - current_relative_offset_; const uint64_t bytes_from_src = std::min(bytes_in_src, bytes_left); - RETURN_NOT_OK(src.read(dest_ptr + dest_offset, bytes_from_src)); + // If the destination pointer is not null, then read into it + // if it is null then we are just seeking + if (dest_ptr != nullptr) + RETURN_NOT_OK(src.read(dest_ptr + dest_offset, bytes_from_src)); bytes_left -= bytes_from_src; dest_offset += bytes_from_src; @@ -114,6 +117,24 @@ Status BufferList::read(void* dest, uint64_t nbytes, uint64_t* bytes_read) { return Status::Ok(); } +Status BufferList::seek(off_t offset, int whence) { + switch (whence) { + case SEEK_SET: + // We just reset the offsets to 0/start, then fall through to seek_current + reset_offset(); + // fall through + case SEEK_CUR: + return read(nullptr, offset); + case SEEK_END: + return Status::BufferError( + "SEEK_END operation not supported for BufferList"); + default: + return Status::BufferError("Invalid seek operation for BufferList"); + } + + return Status::Ok(); +} + void BufferList::reset_offset() { offset_ = 0; current_buffer_index_ = 0; diff --git a/tiledb/sm/buffer/buffer_list.h b/tiledb/sm/buffer/buffer_list.h index f4cd00ce55a..6f283b4f503 100644 --- a/tiledb/sm/buffer/buffer_list.h +++ b/tiledb/sm/buffer/buffer_list.h @@ -82,7 +82,8 @@ class BufferList { * Returns an error if the buffers contain less than the requested number of * bytes starting from the current offset. * - * @param dest The buffer to read the data into. + * @param dest The buffer to read the data into. If null then will perform + * seek. * @param nbytes The number of bytes to read. * @return Status */ @@ -99,6 +100,27 @@ class BufferList { */ Status read_at_most(void* dest, uint64_t nbytes, uint64_t* bytes_read); + /** + * Seek to an offset, similar to lseek or fseek + * + * Whence options are: + * + * SEEK_SET + * The offset is set to offset bytes. + * + * SEEK_CUR + * The offset is set to its current location plus offset bytes + * + * SEEK_END + * This is not supported. Its purpose in lseek would be to set + * the offset to the size of the BufferList plus offset bytes. + * + * @param offset Offset to seek to. + * @param whence Location to seek from. + * @return Status + */ + Status seek(off_t offset, int whence); + /** Resets the current offset for reading. */ void reset_offset(); @@ -121,7 +143,7 @@ class BufferList { /** * Reads from the current offset into the given destination. * - * @param dest The buffer to read the data into. + * @param dest The buffer to read the data into. If null will perform seek. * @param nbytes The number of bytes to read. * @param bytes_read Set to the number of bytes actually read. * @return Status diff --git a/tiledb/sm/rest/curl.cc b/tiledb/sm/rest/curl.cc index 94ebe74b25b..b83c63b9117 100644 --- a/tiledb/sm/rest/curl.cc +++ b/tiledb/sm/rest/curl.cc @@ -161,6 +161,23 @@ size_t buffer_list_read_memory_callback( return num_read; } +/** + * Seek function to handle curl redirects + * @param userp user data (buffer list) + * @param offset offset to seek to + * @param origin whence to seek from + * @return SEEKFUNC status + */ +static int buffer_list_seek_callback( + void* userp, curl_off_t offset, int origin) { + BufferList* data = static_cast(userp); + Status status = data->seek(offset, origin); + if (status.ok()) + return CURL_SEEKFUNC_OK; + + return CURL_SEEKFUNC_FAIL; +} + Curl::Curl() : config_(nullptr) , curl_(nullptr, curl_easy_cleanup) { @@ -522,6 +539,10 @@ Status Curl::post_data_common( /* pass our list of custom made headers */ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, *headers); + /* set seek for handling redirects */ + curl_easy_setopt(curl, CURLOPT_SEEKFUNCTION, &buffer_list_seek_callback); + curl_easy_setopt(curl, CURLOPT_SEEKDATA, data); + return Status::Ok(); }