Skip to content

Commit 5b8e495

Browse files
author
root
committed
release spdk buf
1 parent 67138f2 commit 5b8e495

File tree

1 file changed

+123
-78
lines changed

1 file changed

+123
-78
lines changed

‎module/bdev/passthru/vbdev_passthru.c

+123-78
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ struct vbdev_passthru {
6464
struct spdk_bdev pt_bdev; /* the PT virtual bdev */
6565
struct spdk_io_channel *md_channel;
6666
struct spdk_spinlock used_lock;
67-
uint8_t multiplier;
67+
uint64_t multiplier;
6868
uint32_t md_len;
6969
void *malloc_md_buf;
7070
uint64_t offset_start;
@@ -304,17 +304,21 @@ _pt_complete_io6(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
304304
io_ctx->test);
305305
}
306306

307+
if (request->buf) {
308+
spdk_free(request->buf);
309+
request->buf = NULL;
310+
}
311+
312+
free(request);
307313
_pt_complete_io(bdev_io, success, orig_io);
308314
// spdk_spin_unlock(&pt_node->used_lock);
309-
// free(request->buf);
310-
free(request);
315+
311316
}
312317

313318

314319
static void
315320
_pt_complete_io3(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
316-
{
317-
// struct spdk_bdev_io *orig_io = cb_arg;
321+
{
318322
struct arg_requst *request = (struct arg_requst *)cb_arg;
319323
struct spdk_bdev_io *orig_io = request->bdev_io;
320324
int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED;
@@ -334,9 +338,12 @@ _pt_complete_io3(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
334338

335339
if (!success) {
336340
spdk_bdev_io_complete(orig_io, status);
337-
spdk_bdev_free_io(bdev_io);
338-
// free(request->buf);
341+
if (request->buf) {
342+
spdk_free(request->buf);
343+
request->buf = NULL;
344+
}
339345
free(request);
346+
spdk_bdev_free_io(bdev_io);
340347
return;
341348
}
342349

@@ -350,10 +357,13 @@ _pt_complete_io3(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
350357
break;
351358
case SPDK_BDEV_IO_TYPE_READ:
352359
case SPDK_BDEV_IO_TYPE_WRITE:
353-
if (orig_io->u.bdev.md_buf == NULL) {
354-
_pt_complete_io(bdev_io, success, cb_arg);
355-
// free(request->buf);
360+
if (orig_io->u.bdev.md_buf == NULL) {
361+
if (request->buf) {
362+
spdk_free(request->buf);
363+
request->buf = NULL;
364+
}
356365
free(request);
366+
_pt_complete_io(bdev_io, success, cb_arg);
357367
return;
358368
}
359369
// if(strncmp(request->buf + (get_md_offset(orig_io, pt_node->md_len) % orig_io->bdev->blocklen), pt_node->malloc_md_buf + get_md_offset(orig_io, pt_node->md_len), get_md_count(orig_io, pt_node->md_len)) != 0){
@@ -385,7 +395,10 @@ _pt_complete_io3(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
385395

386396

387397
if (rc != 0) {
388-
// free(request->buf);
398+
if (request->buf) {
399+
spdk_free(request->buf);
400+
request->buf = NULL;
401+
}
389402
free(request);
390403
// if (rc == -ENOMEM) {
391404
// SPDK_ERRLOG("No memory, start to queue io for passthru.\n");
@@ -438,7 +451,8 @@ _pt_complete_io2(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
438451
if(!lb_count) {
439452
lb_count = 1;
440453
}
441-
if(lb + lb_count >= pt_node->offset_start){
454+
455+
if(lb + lb_count >= pt_node->offset_start) {
442456
_pt_complete_io(bdev_io, SPDK_BDEV_IO_STATUS_FAILED, cb_arg);
443457
return;
444458
}
@@ -475,75 +489,72 @@ _pt_complete_io2(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
475489

476490

477491
if (rc != 0) {
478-
free(request->buf);
492+
if (request->buf) {
493+
spdk_free(request->buf);
494+
request->buf = NULL;
495+
}
479496
free(request);
480-
// if (rc == -ENOMEM) {
481-
// SPDK_ERRLOG("No memory, start to queue io for passthru.\n");
482-
// // io_ctx->ch = ch;
483-
// vbdev_passthru_queue_io(orig_io);
484-
// } else {
485497
SPDK_ERRLOG("reading md blocks, ERROR on bdev_io submission!\n");
486-
spdk_bdev_io_complete(orig_io, SPDK_BDEV_IO_STATUS_FAILED);
487-
// }
498+
spdk_bdev_io_complete(orig_io, SPDK_BDEV_IO_STATUS_FAILED);
488499
}
489500
spdk_bdev_free_io(bdev_io);
490501
}
491502

492503

493-
static void
494-
_pt_complete_io5(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
495-
{
496-
// struct spdk_bdev_io *orig_io = cb_arg;
497-
struct arg_requst *request = (struct arg_requst *)cb_arg;
498-
struct spdk_bdev_io *orig_io = request->bdev_io;
499-
int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED;
500-
struct passthru_bdev_io *io_ctx = (struct passthru_bdev_io *)orig_io->driver_ctx;
501-
// struct pt_io_channel *pt_ch = spdk_io_channel_get_ctx(io_ctx->ch);
502-
struct vbdev_passthru *pt_node = SPDK_CONTAINEROF(orig_io->bdev, struct vbdev_passthru, pt_bdev);
503-
504-
/* We setup this value in the submission routine, just showing here that it is
505-
* passed back to us.
506-
*/
507-
if (io_ctx->test != 0x5a) {
508-
SPDK_ERRLOG("Error, original IO device_ctx is wrong! 0x%x\n",
509-
io_ctx->test);
510-
}
511-
512-
if (!success) {
513-
spdk_bdev_io_complete(orig_io, status);
514-
spdk_bdev_free_io(bdev_io);
515-
// free(request->buf);
516-
free(request);
517-
return;
518-
}
519-
520-
if (orig_io->u.bdev.md_buf == NULL) {
521-
_pt_complete_io(bdev_io, success, cb_arg);
522-
// free(request->buf);
523-
free(request);
524-
return;
525-
}
526-
527-
// spdk_spin_lock(&pt_node->used_lock);
528-
// void *md_buf = spdk_zmalloc(get_md_count(orig_io, pt_node->md_len), 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
529-
// memcpy(md_buf, request->buf + (get_md_offset(orig_io, pt_node->md_len) % orig_io->bdev->blocklen), get_md_count(orig_io, pt_node->md_len));
530-
// spdk_bdev_io_set_md_buf(orig_io, md_buf, get_md_count(orig_io, pt_node->md_len));
531-
// memcpy(orig_io->u.bdev.md_buf, (request->buf + (get_md_offset(orig_io, pt_node->md_len)) % orig_io->bdev->blocklen), get_md_count(orig_io, pt_node->md_len));
532-
// memcpy(orig_io->u.bdev.md_buf, pt_node->malloc_md_buf + (get_md_offset(orig_io, pt_node->md_len)), get_md_count(orig_io, pt_node->md_len));
533-
// if(strncmp(request->buf + (get_md_offset(orig_io, pt_node->md_len) % orig_io->bdev->blocklen), pt_node->malloc_md_buf + get_md_offset(orig_io, pt_node->md_len), get_md_count(orig_io, pt_node->md_len)) != 0){
534-
// char *a = request->buf + (get_md_offset(orig_io, pt_node->md_len) % orig_io->bdev->blocklen);
535-
// char *b = pt_node->malloc_md_buf + get_md_offset(orig_io, pt_node->md_len);
536-
// for(int i =0; i < get_md_count(orig_io, pt_node->md_len); i++ ){
537-
// printf("first %02x second %02x \n", (int)a[i], (int)b[i]);
538-
// }
539-
// printf("not equal");
540-
// fflush(stdout);
541-
// }
542-
// spdk_spin_unlock(&pt_node->used_lock);
543-
_pt_complete_io(bdev_io, success, orig_io);
544-
// free(request->buf);
545-
free(request);
546-
}
504+
// static void
505+
// _pt_complete_io5(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
506+
// {
507+
// // struct spdk_bdev_io *orig_io = cb_arg;
508+
// struct arg_requst *request = (struct arg_requst *)cb_arg;
509+
// struct spdk_bdev_io *orig_io = request->bdev_io;
510+
// int status = success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED;
511+
// struct passthru_bdev_io *io_ctx = (struct passthru_bdev_io *)orig_io->driver_ctx;
512+
// // struct pt_io_channel *pt_ch = spdk_io_channel_get_ctx(io_ctx->ch);
513+
// struct vbdev_passthru *pt_node = SPDK_CONTAINEROF(orig_io->bdev, struct vbdev_passthru, pt_bdev);
514+
515+
// /* We setup this value in the submission routine, just showing here that it is
516+
// * passed back to us.
517+
// */
518+
// if (io_ctx->test != 0x5a) {
519+
// SPDK_ERRLOG("Error, original IO device_ctx is wrong! 0x%x\n",
520+
// io_ctx->test);
521+
// }
522+
523+
// if (!success) {
524+
// spdk_bdev_io_complete(orig_io, status);
525+
// spdk_bdev_free_io(bdev_io);
526+
// // free(request->buf);
527+
// free(request);
528+
// return;
529+
// }
530+
531+
// if (orig_io->u.bdev.md_buf == NULL) {
532+
// _pt_complete_io(bdev_io, success, cb_arg);
533+
// // free(request->buf);
534+
// free(request);
535+
// return;
536+
// }
537+
538+
// // spdk_spin_lock(&pt_node->used_lock);
539+
// // void *md_buf = spdk_zmalloc(get_md_count(orig_io, pt_node->md_len), 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA);
540+
// // memcpy(md_buf, request->buf + (get_md_offset(orig_io, pt_node->md_len) % orig_io->bdev->blocklen), get_md_count(orig_io, pt_node->md_len));
541+
// // spdk_bdev_io_set_md_buf(orig_io, md_buf, get_md_count(orig_io, pt_node->md_len));
542+
// // memcpy(orig_io->u.bdev.md_buf, (request->buf + (get_md_offset(orig_io, pt_node->md_len)) % orig_io->bdev->blocklen), get_md_count(orig_io, pt_node->md_len));
543+
// // memcpy(orig_io->u.bdev.md_buf, pt_node->malloc_md_buf + (get_md_offset(orig_io, pt_node->md_len)), get_md_count(orig_io, pt_node->md_len));
544+
// // if(strncmp(request->buf + (get_md_offset(orig_io, pt_node->md_len) % orig_io->bdev->blocklen), pt_node->malloc_md_buf + get_md_offset(orig_io, pt_node->md_len), get_md_count(orig_io, pt_node->md_len)) != 0){
545+
// // char *a = request->buf + (get_md_offset(orig_io, pt_node->md_len) % orig_io->bdev->blocklen);
546+
// // char *b = pt_node->malloc_md_buf + get_md_offset(orig_io, pt_node->md_len);
547+
// // for(int i =0; i < get_md_count(orig_io, pt_node->md_len); i++ ){
548+
// // printf("first %02x second %02x \n", (int)a[i], (int)b[i]);
549+
// // }
550+
// // printf("not equal");
551+
// // fflush(stdout);
552+
// // }
553+
// // spdk_spin_unlock(&pt_node->used_lock);
554+
// _pt_complete_io(bdev_io, success, orig_io);
555+
// // free(request->buf);
556+
// free(request);
557+
// }
547558

548559

549560

@@ -1127,8 +1138,16 @@ vbdev_passthru_register(const char *bdev_name)
11271138
// pt_node->offset_start = (pt_node->offset_start * bdev->blocklen) / pt_node->pt_bdev.blocklen;
11281139
// pt_node->offset_start = 20480;
11291140
pt_node->pt_bdev.blockcnt -= pt_node->offset_start;
1130-
pt_node->malloc_md_buf = spdk_zmalloc((pt_node->pt_bdev.blockcnt * pt_node->md_len) + pt_node->pt_bdev.blocklen, 2 * 1024 * 1024, NULL,
1131-
SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1141+
// pt_node->malloc_md_buf = spdk_zmalloc((pt_node->pt_bdev.blockcnt * pt_node->md_len) + pt_node->pt_bdev.blocklen, 2 * 1024 * 1024, NULL,
1142+
// SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
1143+
1144+
pt_node->malloc_md_buf = spdk_zmalloc(
1145+
(pt_node->offset_start * pt_node->md_len) * pt_node->pt_bdev.blocklen,
1146+
spdk_bdev_get_buf_align(pt_node->base_bdev),
1147+
NULL,
1148+
SPDK_ENV_LCORE_ID_ANY,
1149+
SPDK_MALLOC_DMA
1150+
);
11321151

11331152
// pt_node->malloc_md_buf = spdk_zmalloc(MAX_MD_ALLOC * pt_node->pt_bdev.blocklen, 2 * 1024 * 1024, NULL,
11341153
// SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
@@ -1190,8 +1209,34 @@ vbdev_passthru_register(const char *bdev_name)
11901209
pt_node->offset_start * pt_node->multiplier , _pt_complete_io_zero, NULL);
11911210
}
11921211
else{
1193-
rc = spdk_bdev_read_blocks(pt_node->base_desc, channel->base_ch,
1194-
pt_node->malloc_md_buf, 0, (pt_node->offset_start) * 7, _pt_complete_io_zero, NULL);
1212+
// rc = spdk_bdev_read_blocks(pt_node->base_desc, channel->base_ch,
1213+
// pt_node->malloc_md_buf, 0, (pt_node->offset_start) * pt_node->multiplier, _pt_complete_io_zero, NULL);
1214+
1215+
uint64_t blocks_to_read = 28000 * 8;
1216+
uint64_t max_blocks_per_io = 1024 * 1; // or another value that works
1217+
uint64_t offset = 0;
1218+
while (blocks_to_read > 0) {
1219+
uint64_t blocks_this_io = spdk_min(blocks_to_read, max_blocks_per_io);
1220+
1221+
rc = spdk_bdev_read_blocks(
1222+
pt_node->base_desc,
1223+
channel->base_ch,
1224+
pt_node->malloc_md_buf + offset * pt_node->pt_bdev.blocklen,
1225+
offset,
1226+
blocks_this_io * pt_node->multiplier,
1227+
_pt_complete_io_zero,
1228+
NULL
1229+
);
1230+
1231+
if (rc != 0) {
1232+
SPDK_ERRLOG("Error submitting read I/O\n");
1233+
break;
1234+
}
1235+
1236+
offset += blocks_this_io;
1237+
blocks_to_read -= blocks_this_io;
1238+
}
1239+
11951240
}
11961241
SPDK_NOTICELOG("pt_bdev registered\n");
11971242
SPDK_NOTICELOG("created pt_bdev for: %s\n", name->vbdev_name);

0 commit comments

Comments
 (0)