fix raid concurrency error with priority by setting priority per raid_bdev_io, not in raid_bdev

Author: ashwin-simplyblock

module/bdev/raid/bdev_raid.c

@@ -921,9 +921,7 @@ raid_bdev_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_i
 			  bdev_io->u.bdev.offset_blocks & MASK_OUT_PRIORITY_CLASS, bdev_io->u.bdev.num_blocks,
 			  bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.md_buf,
 			  bdev_io->u.bdev.memory_domain, bdev_io->u.bdev.memory_domain_ctx);
-
-	struct raid_bdev *raid_bdev = raid_io->raid_bdev;
-	raid_bdev->priority_class = (bdev_io->u.bdev.offset_blocks & PRIORITY_CLASS_MASK) >> PRIORITY_CLASS_BITS_POS;
+	raid_io->priority_class = (bdev_io->u.bdev.offset_blocks & PRIORITY_CLASS_MASK) >> PRIORITY_CLASS_BITS_POS;
 
 	switch (bdev_io->type) {
 	case SPDK_BDEV_IO_TYPE_READ:

module/bdev/raid/bdev_raid.h

@@ -126,6 +126,7 @@ struct raid_bdev_io {
 	/* The raid bdev associated with this IO */
 	struct raid_bdev *raid_bdev;
 
+	int8_t priority_class; /* priority class of the parent SPDK blob and lvol who submitted this I/O request */
 	uint64_t offset_blocks;
 	uint64_t num_blocks;
 	struct iovec *iovs;
@@ -194,9 +195,6 @@ struct raid_bdev {
 	/* array of base bdev info */
 	struct raid_base_bdev_info	*base_bdev_info;
 
-	/* priority class of the parent SPDK blob and lvol */
-	int priority_class;
-
 	/* strip size of raid bdev in blocks */
 	uint32_t			strip_size;
 
@@ -406,9 +404,8 @@ static inline int
 raid_bdev_readv_blocks_ext(struct raid_base_bdev_info *base_info, struct spdk_io_channel *ch,
 			   struct iovec *iov, int iovcnt, uint64_t offset_blocks,
 			   uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg,
-			   struct spdk_bdev_ext_io_opts *opts)
+			   struct spdk_bdev_ext_io_opts *opts, int8_t priority_class)
 {
-	const int priority_class = base_info->raid_bdev->priority_class;
 	const uint64_t priority_blocks = ((uint64_t)priority_class << PRIORITY_CLASS_BITS_POS) | (base_info->data_offset + offset_blocks);
 
 	return spdk_bdev_readv_blocks_ext(base_info->desc, ch, iov, iovcnt,
@@ -422,7 +419,7 @@ static inline int
 raid_bdev_writev_blocks_ext(struct raid_base_bdev_info *base_info, struct spdk_io_channel *ch,
 			    struct iovec *iov, int iovcnt, uint64_t offset_blocks,
 			    uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg,
-			    struct spdk_bdev_ext_io_opts *opts)
+			    struct spdk_bdev_ext_io_opts *opts, int8_t priority_class)
 {
 	int rc;
 	uint64_t remapped_offset_blocks = base_info->data_offset + offset_blocks;
@@ -436,7 +433,6 @@ raid_bdev_writev_blocks_ext(struct raid_base_bdev_info *base_info, struct spdk_i
 		}
 	}
 
-	const int priority_class = base_info->raid_bdev->priority_class;
 	const uint64_t priority_blocks = ((uint64_t)priority_class << PRIORITY_CLASS_BITS_POS) | remapped_offset_blocks;
 
 	return spdk_bdev_writev_blocks_ext(base_info->desc, ch, iov, iovcnt,
@@ -451,7 +447,7 @@ raid_bdev_unmap_blocks(struct raid_base_bdev_info *base_info, struct spdk_io_cha
 		       uint64_t offset_blocks, uint64_t num_blocks,
 		       spdk_bdev_io_completion_cb cb, void *cb_arg)
 {
-	const int priority_class = base_info->raid_bdev->priority_class;
+	const int priority_class = ((struct raid_bdev_io*)cb_arg)->priority_class;
 	const uint64_t priority_blocks = ((uint64_t)priority_class << PRIORITY_CLASS_BITS_POS) | (base_info->data_offset + offset_blocks);
 
 	return spdk_bdev_unmap_blocks(base_info->desc, ch, priority_blocks,
@@ -466,7 +462,7 @@ raid_bdev_flush_blocks(struct raid_base_bdev_info *base_info, struct spdk_io_cha
 		       uint64_t offset_blocks, uint64_t num_blocks,
 		       spdk_bdev_io_completion_cb cb, void *cb_arg)
 {
-	const int priority_class = base_info->raid_bdev->priority_class;
+	const int priority_class = ((struct raid_bdev_io*)cb_arg)->priority_class;
 	const uint64_t priority_blocks = ((uint64_t)priority_class << PRIORITY_CLASS_BITS_POS) | (base_info->data_offset + offset_blocks);
 
 	return spdk_bdev_flush_blocks(base_info->desc, ch, priority_blocks,

module/bdev/raid/bdev_raid_sb.c

@@ -359,10 +359,9 @@ _raid_bdev_write_superblock(void *_ctx)
 
 		const uint64_t offset_blocks = 0;
 		const uint64_t num_blocks = raid_bdev->sb_io_buf_size / bdev->blocklen;
-		const uint64_t priority_blocks = ((uint64_t)raid_bdev->priority_class << PRIORITY_CLASS_BITS_POS) | offset_blocks;
 
 		rc = spdk_bdev_write_blocks(base_info->desc, base_info->app_thread_ch,
-				     raid_bdev->sb_io_buf, priority_blocks, num_blocks,
+				     raid_bdev->sb_io_buf, offset_blocks, num_blocks,
 				     raid_bdev_write_superblock_cb, ctx);
 		if (rc != 0) {
 			if (rc == -ENOMEM) {

module/bdev/raid/concat.c

@@ -111,12 +111,12 @@ concat_submit_rw_request(struct raid_bdev_io *raid_io)
 		ret = raid_bdev_readv_blocks_ext(base_info, base_ch,
 						 raid_io->iovs, raid_io->iovcnt,
 						 pd_lba, pd_blocks, concat_bdev_io_completion,
-						 raid_io, &io_opts);
+						 raid_io, &io_opts, raid_io->priority_class);
 	} else if (raid_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
 		ret = raid_bdev_writev_blocks_ext(base_info, base_ch,
 						  raid_io->iovs, raid_io->iovcnt,
 						  pd_lba, pd_blocks, concat_bdev_io_completion,
-						  raid_io, &io_opts);
+						  raid_io, &io_opts, raid_io->priority_class);
 	} else {
 		SPDK_ERRLOG("Recvd not supported io type %u\n", raid_io->type);
 		assert(0);

module/bdev/raid/raid0.c

@@ -127,7 +127,7 @@ raid0_submit_rw_request(struct raid_bdev_io *raid_io)
 		ret = raid_bdev_readv_blocks_ext(base_info, base_ch,
 						 raid_io->iovs, raid_io->iovcnt,
 						 pd_lba, pd_blocks, raid0_bdev_io_completion,
-						 raid_io, &io_opts);
+						 raid_io, &io_opts, raid_io->priority_class);
 	} else if (raid_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
 		struct spdk_bdev *bdev = &base_info->raid_bdev->bdev;
 
@@ -145,7 +145,7 @@ raid0_submit_rw_request(struct raid_bdev_io *raid_io)
 		ret = raid_bdev_writev_blocks_ext(base_info, base_ch,
 						  raid_io->iovs, raid_io->iovcnt,
 						  pd_lba, pd_blocks, raid0_bdev_io_completion,
-						  raid_io, &io_opts);
+						  raid_io, &io_opts, raid_io->priority_class);
 	} else {
 		SPDK_ERRLOG("Recvd not supported io type %u\n", raid_io->type);
 		assert(0);

module/bdev/raid/raid1.c

@@ -113,7 +113,7 @@ raid1_correct_read_error(void *_raid_io)
 	raid1_init_ext_io_opts(&io_opts, raid_io);
 	ret = raid_bdev_writev_blocks_ext(base_info, base_ch, raid_io->iovs, raid_io->iovcnt,
 					  raid_io->offset_blocks, raid_io->num_blocks,
-					  raid1_correct_read_error_completion, raid_io, &io_opts);
+					  raid1_correct_read_error_completion, raid_io, &io_opts, raid_io->priority_class);
 	if (spdk_unlikely(ret != 0)) {
 		if (ret == -ENOMEM) {
 			raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),
@@ -169,7 +169,7 @@ raid1_read_other_base_bdev(void *_raid_io)
 		raid1_init_ext_io_opts(&io_opts, raid_io);
 		ret = raid_bdev_readv_blocks_ext(base_info, base_ch, raid_io->iovs, raid_io->iovcnt,
 						 raid_io->offset_blocks, raid_io->num_blocks,
-						 raid1_read_other_completion, raid_io, &io_opts);
+						 raid1_read_other_completion, raid_io, &io_opts, raid_io->priority_class);
 		if (spdk_unlikely(ret != 0)) {
 			if (ret == -ENOMEM) {
 				raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),
@@ -258,7 +258,7 @@ raid1_submit_read_request(struct raid_bdev_io *raid_io)
 	raid1_init_ext_io_opts(&io_opts, raid_io);
 	ret = raid_bdev_readv_blocks_ext(base_info, base_ch, raid_io->iovs, raid_io->iovcnt,
 					 raid_io->offset_blocks, raid_io->num_blocks,
-					 raid1_read_bdev_io_completion, raid_io, &io_opts);
+					 raid1_read_bdev_io_completion, raid_io, &io_opts, raid_io->priority_class);
 
 	if (spdk_likely(ret == 0)) {
 		raid1_channel_inc_read_counters(raid_ch, idx, raid_io->num_blocks);
@@ -302,7 +302,7 @@ raid1_submit_write_request(struct raid_bdev_io *raid_io)
 
 		ret = raid_bdev_writev_blocks_ext(base_info, base_ch, raid_io->iovs, raid_io->iovcnt,
 						  raid_io->offset_blocks, raid_io->num_blocks,
-						  raid1_write_bdev_io_completion, raid_io, &io_opts);
+						  raid1_write_bdev_io_completion, raid_io, &io_opts, raid_io->priority_class);
 		if (spdk_unlikely(ret != 0)) {
 			if (spdk_unlikely(ret == -ENOMEM)) {
 				raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),
@@ -453,7 +453,7 @@ raid1_process_submit_write(struct raid_bdev_process_request *process_req)
 	ret = raid_bdev_writev_blocks_ext(process_req->target, process_req->target_ch,
 					  raid_io->iovs, raid_io->iovcnt,
 					  raid_io->offset_blocks, raid_io->num_blocks,
-					  raid1_process_write_completed, process_req, &io_opts);
+					  raid1_process_write_completed, process_req, &io_opts, raid_io->priority_class);
 	if (spdk_unlikely(ret != 0)) {
 		if (ret == -ENOMEM) {
 			raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(process_req->target->desc),

module/bdev/raid/raid5f.c

@@ -448,7 +448,7 @@ raid5f_chunk_submit(struct chunk *chunk)
 
 		ret = raid_bdev_writev_blocks_ext(base_info, base_ch, chunk->iovs, chunk->iovcnt,
 						  base_offset_blocks, raid_bdev->strip_size,
-						  raid5f_chunk_complete_bdev_io, chunk, &io_opts);
+						  raid5f_chunk_complete_bdev_io, chunk, &io_opts, raid_io->priority_class);
 		break;
 	case STRIPE_REQ_RECONSTRUCT:
 		if (chunk == stripe_req->reconstruct.chunk) {
@@ -460,7 +460,7 @@ raid5f_chunk_submit(struct chunk *chunk)
 
 		ret = raid_bdev_readv_blocks_ext(base_info, base_ch, chunk->iovs, chunk->iovcnt,
 						 base_offset_blocks, raid_io->num_blocks,
-						 raid5f_chunk_complete_bdev_io, chunk, &io_opts);
+						 raid5f_chunk_complete_bdev_io, chunk, &io_opts, raid_io->priority_class);
 		break;
 	default:
 		assert(false);
@@ -791,7 +791,7 @@ raid5f_submit_read_request(struct raid_bdev_io *raid_io, uint64_t stripe_index,
 
 	ret = raid_bdev_readv_blocks_ext(base_info, base_ch, raid_io->iovs, raid_io->iovcnt,
 					 base_offset_blocks, raid_io->num_blocks,
-					 raid5f_chunk_read_complete, raid_io, &io_opts);
+					 raid5f_chunk_read_complete, raid_io, &io_opts, raid_io->priority_class);
 	if (spdk_unlikely(ret == -ENOMEM)) {
 		raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(base_info->desc),
 					base_ch, _raid5f_submit_rw_request);
@@ -1174,7 +1174,7 @@ raid5f_process_submit_write(struct raid_bdev_process_request *process_req)
 	ret = raid_bdev_writev_blocks_ext(process_req->target, process_req->target_ch,
 					  raid_io->iovs, raid_io->iovcnt,
 					  stripe_index << raid_bdev->strip_size_shift, raid_bdev->strip_size,
-					  raid5f_process_write_completed, process_req, &io_opts);
+					  raid5f_process_write_completed, process_req, &io_opts, raid_io->priority_class);
 	if (spdk_unlikely(ret != 0)) {
 		if (ret == -ENOMEM) {
 			raid_bdev_queue_io_wait(raid_io, spdk_bdev_desc_get_bdev(process_req->target->desc),