Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Coordinate split before every compression (and zip after decompression) #7

Merged
merged 1 commit into from
May 18, 2017
Merged

Coordinate split before every compression (and zip after decompression) #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions core/include/fragment/read_state.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,7 @@ class ReadState {
/**
* Decompresses a tile with GZIP.
*
* @param attribute_id The attribute id.
* @param tile_compressed The compressed tile to be decompressed.
* @param tile_compressed_size The size of the compressed tile.
* @param tile The resulting decompressed tile.
Expand All @@ -606,6 +607,7 @@ class ReadState {
* @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error.
*/
int decompress_tile_gzip(
int attribute_id,
unsigned char* tile_compressed,
size_t tile_compressed_size,
unsigned char* tile,
Expand All @@ -614,6 +616,7 @@ class ReadState {
/**
* Decompresses a tile with Zstandard.
*
* @param attribute_id The attribute id.
* @param tile_compressed The compressed tile to be decompressed.
* @param tile_compressed_size The size of the compressed tile.
* @param tile The resulting decompressed tile.
Expand All @@ -622,6 +625,7 @@ class ReadState {
* @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error.
*/
int decompress_tile_zstd(
int attribute_id,
unsigned char* tile_compressed,
size_t tile_compressed_size,
unsigned char* tile,
Expand All @@ -630,6 +634,7 @@ class ReadState {
/**
* Decompresses a tile with LZ4.
*
* @param attribute_id The attribute id.
* @param tile_compressed The compressed tile to be decompressed.
* @param tile_compressed_size The size of the compressed tile.
* @param tile The resulting decompressed tile.
Expand All @@ -638,6 +643,7 @@ class ReadState {
* @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error.
*/
int decompress_tile_lz4(
int attribute_id,
unsigned char* tile_compressed,
size_t tile_compressed_size,
unsigned char* tile,
Expand All @@ -646,6 +652,7 @@ class ReadState {
/**
* Decompresses a tile with Blosc.
*
* @param attribute_id The attribute id.
* @param tile_compressed The compressed tile to be decompressed.
* @param tile_compressed_size The size of the compressed tile.
* @param tile The resulting decompressed tile.
Expand All @@ -655,6 +662,7 @@ class ReadState {
* @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error.
*/
int decompress_tile_blosc(
int attribute_id,
unsigned char* tile_compressed,
size_t tile_compressed_size,
unsigned char* tile,
Expand Down Expand Up @@ -682,6 +690,7 @@ class ReadState {
/**
* Decompresses a tile with BZIP2.
*
* @param attribute_id The attribute id.
* @param tile_compressed The compressed tile to be decompressed.
* @param tile_compressed_size The size of the compressed tile.
* @param tile The resulting decompressed tile.
Expand All @@ -690,6 +699,7 @@ class ReadState {
* @return TILEDB_RS_OK for success and TILEDB_RS_ERR for error.
*/
int decompress_tile_bzip2(
int attribute_id,
unsigned char* tile_compressed,
size_t tile_compressed_size,
unsigned char* tile,
Expand Down
8 changes: 8 additions & 0 deletions core/include/fragment/write_state.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,12 +226,14 @@ class WriteState {
* Compresses with GZIP the input tile buffer, and stores it inside
* tile_compressed_ member attribute.
*
* @param attribute_id The attribute id.
* @param tile The tile buffer to be compressed.
* @param tile_size The size of the tile buffer in bytes.
* @param tile_compressed_size The size of the resulting compressed tile.
* @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error.
*/
int compress_tile_gzip(
int attribute_id,
unsigned char* tile,
size_t tile_size,
size_t& tile_compressed_size);
Expand All @@ -240,12 +242,14 @@ class WriteState {
* Compresses with Zstandard the input tile buffer, and stores it inside
* tile_compressed_ member attribute.
*
* @param attribute_id The attribute id.
* @param tile The tile buffer to be compressed.
* @param tile_size The size of the tile buffer in bytes.
* @param tile_compressed_size The size of the resulting compressed tile.
* @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error.
*/
int compress_tile_zstd(
int attribute_id,
unsigned char* tile,
size_t tile_size,
size_t& tile_compressed_size);
Expand All @@ -254,12 +258,14 @@ class WriteState {
* Compresses with LZ4 the input tile buffer, and stores it inside
* tile_compressed_ member attribute.
*
* @param attribute_id The attribute id.
* @param tile The tile buffer to be compressed.
* @param tile_size The size of the tile buffer in bytes.
* @param tile_compressed_size The size of the resulting compressed tile.
* @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error.
*/
int compress_tile_lz4(
int attribute_id,
unsigned char* tile,
size_t tile_size,
size_t& tile_compressed_size);
Expand Down Expand Up @@ -302,12 +308,14 @@ class WriteState {
* Compresses with BZIP2 the input tile buffer, and stores it inside
* tile_compressed_ member attribute.
*
* @param attribute_id The attribute id.
* @param tile The tile buffer to be compressed.
* @param tile_size The size of the tile buffer in bytes.
* @param tile_compressed_size The size of the resulting compressed tile.
* @return TILEDB_WS_OK on success and TILEDB_WS_ERR on error.
*/
int compress_tile_bzip2(
int attribute_id,
unsigned char* tile,
size_t tile_size,
size_t& tile_compressed_size);
Expand Down
31 changes: 31 additions & 0 deletions core/include/misc/utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -748,6 +748,21 @@ int RLE_decompress_coords_row(
size_t value_size,
int dim_num);

/**
* Receives a tile of coordinate tuples and splits it across the dimensions.
*
* @param tile The input tile.
* @param tile_size The tile size.
* @param dim_num The number of dimensions.
* @param coords_size The size in bytes of a tuple of coordinates.
* @return void
*/
void split_coordinates(
void* tile,
size_t tile_size,
int dim_num,
size_t coords_size);

/**
* Checks if a string starts with a certain prefix.
*
Expand Down Expand Up @@ -792,4 +807,20 @@ int write_to_file_cmp_gzip(
const void* buffer,
size_t buffer_size);

/**
* Creates tuples of coordinates from vertically partitioned dimensions.
*
* @param tile The input tile.
* @param tile_size The tile size.
* @param dim_num The number of dimensions.
* @param coords_size The size in bytes of a tuple of coordinates.
* @return void
*/
void zip_coordinates(
void* tile,
size_t tile_size,
int dim_num,
size_t coords_size);


#endif
Loading