Skip to content

Commit

Permalink
[feature] Add stopping points to isal_inflate
Browse files Browse the repository at this point in the history
  • Loading branch information
@mxmlnkn
mxmlnkn committed Aug 26, 2023
1 parent afff478 commit 9e1dea6
Show file tree
Hide file tree
Showing 2 changed files with 135 additions and 6 deletions.
125 changes: 119 additions & 6 deletions igzip/igzip_inflate.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1444,6 +1444,7 @@ static int read_header(struct inflate_state *state)

state->bfinal = inflate_in_read_bits(state, 1);
btype = inflate_in_read_bits(state, 2);
state->btype = btype;

if (state->read_in_length < 0)
ret = ISAL_END_INPUT;
Expand Down Expand Up @@ -1778,6 +1779,9 @@ void isal_inflate_init(struct inflate_state *state)
state->tmp_in_size = 0;
state->tmp_out_processed = 0;
state->tmp_out_valid = 0;
state->points_to_stop_at = ISAL_STOPPING_POINT_NONE;
state->stopped_at = ISAL_STOPPING_POINT_NONE;
state->btype = -1;
}

void isal_inflate_reset(struct inflate_state *state)
Expand All @@ -1798,6 +1802,8 @@ void isal_inflate_reset(struct inflate_state *state)
state->tmp_in_size = 0;
state->tmp_out_processed = 0;
state->tmp_out_valid = 0;
state->stopped_at = ISAL_STOPPING_POINT_NONE;
state->btype = -1;
}

static inline uint32_t fixed_size_read(struct inflate_state *state,
Expand Down Expand Up @@ -2278,6 +2284,8 @@ int isal_inflate(struct inflate_state *state)
int32_t shift_size = 0;
int ret = 0;

state->stopped_at = ISAL_STOPPING_POINT_NONE;

if (!state->wrapper_flag && state->crc_flag == IGZIP_GZIP) {
struct isal_gzip_header gz_hdr;
isal_gzip_header_init(&gz_hdr);
Expand All @@ -2286,6 +2294,12 @@ int isal_inflate(struct inflate_state *state)
return ret;
else if (ret > 0)
return ISAL_DECOMP_OK;

if ((ret == 0) && (state->points_to_stop_at & ISAL_STOPPING_POINT_END_OF_STREAM_HEADER)) {
state->stopped_at = ISAL_STOPPING_POINT_END_OF_STREAM_HEADER;
return ISAL_DECOMP_OK;
}

} else if (!state->wrapper_flag && state->crc_flag == IGZIP_ZLIB) {
struct isal_zlib_header z_hdr = { 0 };
ret = isal_read_zlib_header(state, &z_hdr);
Expand All @@ -2298,6 +2312,11 @@ int isal_inflate(struct inflate_state *state)
state->dict_id = z_hdr.dict_id;
return ISAL_NEED_DICT;
}

if ((ret == 0) && (state->points_to_stop_at & ISAL_STOPPING_POINT_END_OF_STREAM_HEADER)) {
state->stopped_at = ISAL_STOPPING_POINT_END_OF_STREAM_HEADER;
return ISAL_DECOMP_OK;
}
} else if (state->block_state == ISAL_CHECKSUM_CHECK) {
switch (state->crc_flag) {
case ISAL_ZLIB:
Expand All @@ -2313,6 +2332,23 @@ int isal_inflate(struct inflate_state *state)
return (ret > 0) ? ISAL_DECOMP_OK : ret;
}

/* This is used to implement the stopping point feature. In order to exist the complex loop,
* it simply saves off all input buffer values to feign having run out of input data.
* Before returning, the buffers are then restored. */
int read_buffer_has_been_saved = 0;
uint8_t *next_in = NULL;
uint64_t read_in = 0;
uint32_t avail_in = 0;
int32_t read_in_length = 0;
int16_t tmp_in_size = 0;
uint8_t tmp_in_buffer[ISAL_DEF_MAX_HDR_SIZE];

/* These are used to determine whether a call made progress to decide whether a stopping point
* request needs to be executed. */
int32_t old_read_in_length = 0;
uint32_t old_avail_in = 0;
int made_progress = 0;

if (state->block_state != ISAL_BLOCK_FINISH) {
state->total_out += state->tmp_out_valid - state->tmp_out_processed;
/* If space in tmp_out buffer, decompress into the tmp_out_buffer */
Expand All @@ -2330,23 +2366,56 @@ int isal_inflate(struct inflate_state *state)
while (state->block_state != ISAL_BLOCK_INPUT_DONE) {
if (state->block_state == ISAL_BLOCK_NEW_HDR
|| state->block_state == ISAL_BLOCK_HDR) {
ret = read_header_stateful(state);
old_read_in_length = state->read_in_length;
old_avail_in = state->avail_in;

/* Will also return 0 if it hasn't read anything, it seems. */
ret = read_header_stateful(state);
made_progress = (old_read_in_length != state->read_in_length) || (old_avail_in != state->avail_in);
if (made_progress && (ret == 0)
&& (state->points_to_stop_at & ISAL_STOPPING_POINT_END_OF_BLOCK_HEADER)) {
state->stopped_at = ISAL_STOPPING_POINT_END_OF_BLOCK_HEADER;
break;
}
if (ret)
break;
}

old_read_in_length = state->read_in_length;
old_avail_in = state->avail_in;

if (state->block_state == ISAL_BLOCK_TYPE0) {
ret = decode_literal_block(state);
} else {
uint8_t *tmp = state->tmp_out_buffer;
ret = decode_huffman_code_block_stateless(state, tmp);
}

made_progress = (old_read_in_length != state->read_in_length) || (old_avail_in != state->avail_in);
if (made_progress && (ret == 0) && (state->points_to_stop_at & ISAL_STOPPING_POINT_END_OF_BLOCK)) {
state->stopped_at = ISAL_STOPPING_POINT_END_OF_BLOCK;
break;
}

if (ret)
break;
}

if (!read_buffer_has_been_saved && (state->stopped_at != ISAL_STOPPING_POINT_NONE)) {
read_buffer_has_been_saved = 1;

next_in = state->next_in;
read_in = state->read_in;
avail_in = state->avail_in;
read_in_length = state->read_in_length;
tmp_in_size = state->tmp_in_size;
memcpy(tmp_in_buffer, state->tmp_in_buffer, sizeof(tmp_in_buffer));

state->avail_in = 0;
state->read_in_length = 0;
state->tmp_in_size = 0;
}

/* Copy valid data from internal buffer into out_buffer */
if (state->write_overflow_len != 0) {
store_le_u32(state->next_out, state->write_overflow_lits);
Expand Down Expand Up @@ -2391,7 +2460,7 @@ int isal_inflate(struct inflate_state *state)
state->total_out -= state->tmp_out_valid - state->tmp_out_processed;
if (state->crc_flag)
update_checksum(state, start_out, state->next_out - start_out);
return ret;
goto stop_inflate_and_return;
}

/* If all data from tmp_out buffer has been processed, start
Expand All @@ -2400,20 +2469,53 @@ int isal_inflate(struct inflate_state *state)
while (state->block_state != ISAL_BLOCK_INPUT_DONE) {
if (state->block_state == ISAL_BLOCK_NEW_HDR
|| state->block_state == ISAL_BLOCK_HDR) {
old_read_in_length = state->read_in_length;
old_avail_in = state->avail_in;

ret = read_header_stateful(state);
if (ret)
break;

made_progress = (old_read_in_length != state->read_in_length) || (old_avail_in != state->avail_in);
if (made_progress && (ret == 0)
&& (state->points_to_stop_at & ISAL_STOPPING_POINT_END_OF_BLOCK_HEADER)) {
state->stopped_at = ISAL_STOPPING_POINT_END_OF_BLOCK_HEADER;
break;
}
}

old_read_in_length = state->read_in_length;
old_avail_in = state->avail_in;

if (state->block_state == ISAL_BLOCK_TYPE0)
ret = decode_literal_block(state);
else
ret =
decode_huffman_code_block_stateless(state,
start_out);
ret = decode_huffman_code_block_stateless(state, start_out);

made_progress = (old_read_in_length != state->read_in_length) || (old_avail_in != state->avail_in);
if (made_progress && (ret == 0) && (state->points_to_stop_at & ISAL_STOPPING_POINT_END_OF_BLOCK)) {
state->stopped_at = ISAL_STOPPING_POINT_END_OF_BLOCK;
break;
}

if (ret)
break;
}

if (!read_buffer_has_been_saved && (state->stopped_at != ISAL_STOPPING_POINT_NONE)) {
read_buffer_has_been_saved = 1;

next_in = state->next_in;
read_in = state->read_in;
avail_in = state->avail_in;
read_in_length = state->read_in_length;
tmp_in_size = state->tmp_in_size;
memcpy(tmp_in_buffer, state->tmp_in_buffer, sizeof(tmp_in_buffer));

state->avail_in = 0;
state->read_in_length = 0;
state->tmp_in_size = 0;
}
}

if (state->crc_flag)
Expand Down Expand Up @@ -2467,7 +2569,7 @@ int isal_inflate(struct inflate_state *state)
if (ret == ISAL_INVALID_LOOKBACK || ret == ISAL_INVALID_BLOCK
|| ret == ISAL_INVALID_SYMBOL) {
state->total_out -= state->tmp_out_valid - state->tmp_out_processed;
return ret;
goto stop_inflate_and_return;
}

if (state->block_state == ISAL_BLOCK_INPUT_DONE
Expand Down Expand Up @@ -2495,5 +2597,16 @@ int isal_inflate(struct inflate_state *state)
state->total_out -= state->tmp_out_valid - state->tmp_out_processed;
}

stop_inflate_and_return:

if (read_buffer_has_been_saved) {
state->next_in = next_in;
state->read_in = read_in;
state->avail_in = avail_in;
state->read_in_length = read_in_length;
state->tmp_in_size = tmp_in_size;
memcpy(state->tmp_in_buffer, tmp_in_buffer, sizeof(tmp_in_buffer));
}

return (ret > 0) ? ISAL_DECOMP_OK : ret;
}
16 changes: 16 additions & 0 deletions include/igzip_lib.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,17 @@ enum isal_block_state {
};


enum isal_stopping_point
{
ISAL_STOPPING_POINT_NONE = 0,
ISAL_STOPPING_POINT_END_OF_STREAM_HEADER = 1U << 0U,
ISAL_STOPPING_POINT_END_OF_STREAM = 1U << 1U, // after gzip footer has been read
ISAL_STOPPING_POINT_END_OF_BLOCK_HEADER = 1U << 2U,
ISAL_STOPPING_POINT_END_OF_BLOCK = 1U << 3U,
ISAL_STOPPING_POINT_ALL = 0xFFFFFFFFU,
};


/* Inflate Flags */
#define ISAL_DEFLATE 0 /* Default */
#define ISAL_GZIP 1
Expand Down Expand Up @@ -511,6 +522,11 @@ struct inflate_state {
int32_t tmp_out_processed; //!< Number of bytes processed in tmp_out_buffer
uint8_t tmp_in_buffer[ISAL_DEF_MAX_HDR_SIZE]; //!< Temporary buffer containing data from the input stream
uint8_t tmp_out_buffer[2 * ISAL_DEF_HIST_SIZE + ISAL_LOOK_AHEAD]; //!< Temporary buffer containing data from the output stream

enum isal_stopping_point points_to_stop_at;
enum isal_stopping_point stopped_at;
/* Only has a meaningful value when stopped_at == ISAL_STOPPING_POINT_END_OF_BLOCK_HEADER. */
uint8_t btype;
};

/******************************************************************************/
Expand Down

0 comments on commit 9e1dea6

Please sign in to comment.