Skip to content

Commit

Permalink
compress: change zio_compress API to use ABDs
Browse files Browse the repository at this point in the history 
This commit changes the frontend zio_compress_data and
zio_decompress_data APIs to take ABD points instead of buffer pointers.

All callers are updated to match. Any that already have an appropriate
ABD nearby now use it directly, while at the rest we create an one.

Internally, the ABDs are passed through to the provider directly.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Signed-off-by: Rob Norris <[email protected]>
  • Loading branch information
@robn @tonyhutter
robn authored and tonyhutter committed Aug 22, 2024
1 parent d3c1238 commit f62e6e1
Show file tree
Hide file tree
Showing 10 changed files with 116 additions and 112 deletions.
37 changes: 25 additions & 12 deletions cmd/zdb/zdb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4657,7 +4657,6 @@ dump_l2arc_log_blocks(int fd, const l2arc_dev_hdr_phys_t *l2dhdr,
l2arc_log_blk_phys_t this_lb;
uint64_t asize;
l2arc_log_blkptr_t lbps[2];
abd_t *abd;
zio_cksum_t cksum;
int failed = 0;
l2arc_dev_t dev;
Expand Down Expand Up @@ -4711,20 +4710,25 @@ dump_l2arc_log_blocks(int fd, const l2arc_dev_hdr_phys_t *l2dhdr,
switch (L2BLK_GET_COMPRESS((&lbps[0])->lbp_prop)) {
case ZIO_COMPRESS_OFF:
break;
default:
abd = abd_alloc_for_io(asize, B_TRUE);
default: {
abd_t *abd = abd_alloc_linear(asize, B_TRUE);
abd_copy_from_buf_off(abd, &this_lb, 0, asize);
if (zio_decompress_data(L2BLK_GET_COMPRESS(
(&lbps[0])->lbp_prop), abd, &this_lb,
asize, sizeof (this_lb), NULL) != 0) {
abd_t dabd;
abd_get_from_buf_struct(&dabd, &this_lb,
sizeof (this_lb));
int err = zio_decompress_data(L2BLK_GET_COMPRESS(
(&lbps[0])->lbp_prop), abd, &dabd,
asize, sizeof (this_lb), NULL);
abd_free(&dabd);
abd_free(abd);
if (err != 0) {
(void) printf("L2ARC block decompression "
"failed\n");
abd_free(abd);
goto out;
}
abd_free(abd);
break;
}
}

if (this_lb.lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
byteswap_uint64_array(&this_lb, sizeof (this_lb));
Expand Down Expand Up @@ -8618,13 +8622,22 @@ try_decompress_block(abd_t *pabd, uint64_t lsize, uint64_t psize,
memset(lbuf, 0x00, lsize);
memset(lbuf2, 0xff, lsize);

abd_t labd, labd2;
abd_get_from_buf_struct(&labd, lbuf, lsize);
abd_get_from_buf_struct(&labd2, lbuf2, lsize);

boolean_t ret = B_FALSE;
if (zio_decompress_data(cfunc, pabd,
lbuf, psize, lsize, NULL) == 0 &&
&labd, psize, lsize, NULL) == 0 &&
zio_decompress_data(cfunc, pabd,
lbuf2, psize, lsize, NULL) == 0 &&
&labd2, psize, lsize, NULL) == 0 &&
memcmp(lbuf, lbuf2, lsize) == 0)
return (B_TRUE);
return (B_FALSE);
ret = B_TRUE;

abd_free(&labd2);
abd_free(&labd);

return (ret);
}

static uint64_t
Expand Down
17 changes: 11 additions & 6 deletions cmd/zstream/zstream_recompress.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,12 +259,13 @@ zstream_do_recompress(int argc, char *argv[])
/* Read and decompress the payload */
(void) sfread(cbuf, payload_size, stdin);
if (dtype != ZIO_COMPRESS_OFF) {
abd_t cabd;
abd_t cabd, dabd;
abd_get_from_buf_struct(&cabd,
cbuf, payload_size);
if (zio_decompress_data(dtype, &cabd, dbuf,
payload_size,
MIN(bufsz, drrw->drr_logical_size),
abd_get_from_buf_struct(&dabd, dbuf,
MIN(bufsz, drrw->drr_logical_size));
if (zio_decompress_data(dtype, &cabd, &dabd,
payload_size, abd_get_size(&dabd),
NULL) != 0) {
warnx("decompression type %d failed "
"for ino %llu offset %llu",
Expand All @@ -274,17 +275,20 @@ zstream_do_recompress(int argc, char *argv[])
exit(4);
}
payload_size = drrw->drr_logical_size;
abd_free(&dabd);
abd_free(&cabd);
free(cbuf);
}

/* Recompress the payload */
if (ctype != ZIO_COMPRESS_OFF) {
abd_t dabd;
abd_t dabd, abd;
abd_get_from_buf_struct(&dabd,
dbuf, drrw->drr_logical_size);
abd_t *pabd =
abd_get_from_buf_struct(&abd, buf, bufsz);
payload_size = P2ROUNDUP(zio_compress_data(
ctype, &dabd, (void **)&buf,
ctype, &dabd, &pabd,
drrw->drr_logical_size, level),
SPA_MINBLOCKSIZE);
if (payload_size != drrw->drr_logical_size) {
Expand All @@ -296,6 +300,7 @@ zstream_do_recompress(int argc, char *argv[])
drrw->drr_compressiontype = 0;
drrw->drr_compressed_size = 0;
}
abd_free(&abd);
abd_free(&dabd);
free(dbuf);
} else {
Expand Down
4 changes: 2 additions & 2 deletions include/sys/zio_compress.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,9 +173,9 @@ extern int zfs_lz4_decompress(abd_t *src, abd_t *dst, size_t s_len,
/*
* Compress and decompress data if necessary.
*/
extern size_t zio_compress_data(enum zio_compress c, abd_t *src, void **dst,
extern size_t zio_compress_data(enum zio_compress c, abd_t *src, abd_t **dst,
size_t s_len, uint8_t level);
extern int zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
extern int zio_decompress_data(enum zio_compress c, abd_t *src, abd_t *abd,
size_t s_len, size_t d_len, uint8_t *level);
extern int zio_compress_to_feature(enum zio_compress comp);

Expand Down
84 changes: 37 additions & 47 deletions module/zfs/arc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1767,12 +1767,12 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
uint64_t csize;
uint64_t lsize = HDR_GET_LSIZE(hdr);
uint64_t psize = HDR_GET_PSIZE(hdr);
void *tmpbuf = NULL;
abd_t *abd = hdr->b_l1hdr.b_pabd;
boolean_t free_abd = B_FALSE;

ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
ASSERT(HDR_AUTHENTICATED(hdr));
ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
ASSERT3P(abd, !=, NULL);

/*
* The MAC is calculated on the compressed data that is stored on disk.
Expand All @@ -1784,14 +1784,13 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
*/
if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
!HDR_COMPRESSION_ENABLED(hdr)) {

abd = NULL;
csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
hdr->b_l1hdr.b_pabd, &tmpbuf, lsize, hdr->b_complevel);
ASSERT3P(tmpbuf, !=, NULL);
hdr->b_l1hdr.b_pabd, &abd, lsize, hdr->b_complevel);
ASSERT3P(abd, !=, NULL);
ASSERT3U(csize, <=, psize);
abd = abd_get_from_buf(tmpbuf, lsize);
abd_take_ownership_of_buf(abd, B_TRUE);
abd_zero_off(abd, csize, psize - csize);
free_abd = B_TRUE;
}

/*
Expand All @@ -1810,16 +1809,10 @@ arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)

if (ret == 0)
arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
else if (ret != ENOENT)
goto error;
else if (ret == ENOENT)
ret = 0;

if (tmpbuf != NULL)
abd_free(abd);

return (0);

error:
if (tmpbuf != NULL)
if (free_abd)
abd_free(abd);

return (ret);
Expand All @@ -1836,7 +1829,6 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
{
int ret;
abd_t *cabd = NULL;
void *tmp = NULL;
boolean_t no_crypt = B_FALSE;
boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);

Expand Down Expand Up @@ -1871,17 +1863,14 @@ arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
* linear buffer and wrapping it in an abd later.
*/
cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));

ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
HDR_GET_LSIZE(hdr), &hdr->b_complevel);
if (ret != 0) {
abd_return_buf(cabd, tmp, arc_hdr_size(hdr));
goto error;
}

abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
arc_hdr_size(hdr), hdr);
hdr->b_l1hdr.b_pabd = cabd;
Expand Down Expand Up @@ -2123,10 +2112,14 @@ arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
/* Skip byteswapping and checksumming (already done) */
return (0);
} else {
abd_t dabd;
abd_get_from_buf_struct(&dabd, buf->b_data,
HDR_GET_LSIZE(hdr));
error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
hdr->b_l1hdr.b_pabd, buf->b_data,
hdr->b_l1hdr.b_pabd, &dabd,
HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
&hdr->b_complevel);
abd_free(&dabd);

/*
* Absent hardware errors or software bugs, this should
Expand Down Expand Up @@ -8531,18 +8524,15 @@ l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
!HDR_COMPRESSION_ENABLED(hdr)) {
abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
ARC_HDR_USE_RESERVE);
void *tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));

ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
HDR_GET_LSIZE(hdr), &hdr->b_complevel);
if (ret != 0) {
abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
goto error;
}

abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
arc_hdr_size(hdr), hdr);
hdr->b_l1hdr.b_pabd = cabd;
Expand Down Expand Up @@ -9037,23 +9027,21 @@ l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
}

if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
size_t bufsize = MAX(size, asize);
void *buf = zio_buf_alloc(bufsize);
uint64_t csize = zio_compress_data(compress, to_write, &buf,
cabd = abd_alloc_for_io(MAX(size, asize), ismd);
uint64_t csize = zio_compress_data(compress, to_write, &cabd,
size, hdr->b_complevel);
if (csize > psize) {
/*
* We can't re-compress the block into the original
* psize. Even if it fits into asize, it does not
* matter, since checksum will never match on read.
*/
zio_buf_free(buf, bufsize);
abd_free(cabd);
return (SET_ERROR(EIO));
}
if (asize > csize)
memset((char *)buf + csize, 0, asize - csize);
to_write = cabd = abd_get_from_buf(buf, bufsize);
abd_take_ownership_of_buf(cabd, B_TRUE);
abd_zero_off(cabd, csize, asize - csize);
to_write = cabd;
}

if (HDR_ENCRYPTED(hdr)) {
Expand Down Expand Up @@ -10184,7 +10172,6 @@ l2arc_log_blk_read(l2arc_dev_t *dev,
{
int err = 0;
zio_cksum_t cksum;
abd_t *abd = NULL;
uint64_t asize;

ASSERT(this_lbp != NULL && next_lbp != NULL);
Expand Down Expand Up @@ -10246,16 +10233,22 @@ l2arc_log_blk_read(l2arc_dev_t *dev,
switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
case ZIO_COMPRESS_OFF:
break;
case ZIO_COMPRESS_LZ4:
abd = abd_alloc_for_io(asize, B_TRUE);
case ZIO_COMPRESS_LZ4: {
abd_t *abd = abd_alloc_linear(asize, B_TRUE);
abd_copy_from_buf_off(abd, this_lb, 0, asize);
if ((err = zio_decompress_data(
abd_t dabd;
abd_get_from_buf_struct(&dabd, this_lb, sizeof (*this_lb));
err = zio_decompress_data(
L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
abd, this_lb, asize, sizeof (*this_lb), NULL)) != 0) {
abd, &dabd, asize, sizeof (*this_lb), NULL);
abd_free(&dabd);
abd_free(abd);
if (err != 0) {
err = SET_ERROR(EINVAL);
goto cleanup;
}
break;
}
default:
err = SET_ERROR(EINVAL);
goto cleanup;
Expand All @@ -10272,8 +10265,6 @@ l2arc_log_blk_read(l2arc_dev_t *dev,
l2arc_log_blk_fetch_abort(*next_io);
*next_io = NULL;
}
if (abd != NULL)
abd_free(abd);
return (err);
}

Expand Down Expand Up @@ -10509,7 +10500,7 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
uint64_t psize, asize;
zio_t *wzio;
l2arc_lb_abd_buf_t *abd_buf;
uint8_t *tmpbuf = NULL;
abd_t *abd = NULL;
l2arc_lb_ptr_buf_t *lb_ptr_buf;

VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);
Expand All @@ -10532,7 +10523,7 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)

/* try to compress the buffer */
psize = zio_compress_data(ZIO_COMPRESS_LZ4,
abd_buf->abd, (void **) &tmpbuf, sizeof (*lb), 0);
abd_buf->abd, &abd, sizeof (*lb), 0);

/* a log block is never entirely zero */
ASSERT(psize != 0);
Expand All @@ -10558,27 +10549,26 @@ l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
ZIO_CHECKSUM_FLETCHER_4);
if (asize < sizeof (*lb)) {
/* compression succeeded */
memset(tmpbuf + psize, 0, asize - psize);
abd_zero_off(abd, psize, asize - psize);
L2BLK_SET_COMPRESS(
(&l2dhdr->dh_start_lbps[0])->lbp_prop,
ZIO_COMPRESS_LZ4);
} else {
/* compression failed */
memcpy(tmpbuf, lb, sizeof (*lb));
abd_copy_from_buf_off(abd, lb, 0, sizeof (*lb));
L2BLK_SET_COMPRESS(
(&l2dhdr->dh_start_lbps[0])->lbp_prop,
ZIO_COMPRESS_OFF);
}

/* checksum what we're about to write */
fletcher_4_native(tmpbuf, asize, NULL,
abd_fletcher_4_native(abd, asize, NULL,
&l2dhdr->dh_start_lbps[0].lbp_cksum);

abd_free(abd_buf->abd);

/* perform the write itself */
abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb));
abd_take_ownership_of_buf(abd_buf->abd, B_TRUE);
abd_buf->abd = abd;
wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE);
Expand Down
12 changes: 7 additions & 5 deletions module/zfs/blkptr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,11 +142,13 @@ decode_embedded_bp(const blkptr_t *bp, void *buf, int buflen)
if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) {
uint8_t dstbuf[BPE_PAYLOAD_SIZE];
decode_embedded_bp_compressed(bp, dstbuf);
abd_t dstabd;
abd_get_from_buf_struct(&dstabd, dstbuf, psize);
VERIFY0(zio_decompress_data(BP_GET_COMPRESS(bp), &dstabd,
buf, psize, buflen, NULL));
abd_free(&dstabd);
abd_t cabd, dabd;
abd_get_from_buf_struct(&cabd, dstbuf, psize);
abd_get_from_buf_struct(&dabd, buf, buflen);
VERIFY0(zio_decompress_data(BP_GET_COMPRESS(bp), &cabd,
&dabd, psize, buflen, NULL));
abd_free(&dabd);
abd_free(&cabd);
} else {
ASSERT3U(lsize, ==, psize);
decode_embedded_bp_compressed(bp, buf);
Expand Down
Loading

0 comments on commit f62e6e1

Please sign in to comment.