diff --git a/include/spdk/nvmf.h b/include/spdk/nvmf.h index dfc0e066911..4efec8eec2b 100644 --- a/include/spdk/nvmf.h +++ b/include/spdk/nvmf.h @@ -125,6 +125,7 @@ struct spdk_nvmf_io_pacer_stat { uint64_t calls; uint64_t no_ios; uint64_t period_ticks; + uint64_t nos_credit_unavailable; }; struct spdk_nvmf_transport_poll_group_stat { diff --git a/lib/nvmf/io_pacer.c b/lib/nvmf/io_pacer.c index f27448ed2ce..d3622cc6d68 100644 --- a/lib/nvmf/io_pacer.c +++ b/lib/nvmf/io_pacer.c @@ -57,6 +57,8 @@ struct io_pacer_queue { STAILQ_HEAD(, io_pacer_queue_entry) queue; }; +struct spdk_io_pacer_tuner; +struct spdk_io_pacer_tuner2; struct spdk_io_pacer { uint64_t period_ticks; int64_t credit; @@ -72,6 +74,11 @@ struct spdk_io_pacer { struct io_pacer_queue *queues; struct spdk_poller *poller; uint32_t disk_credit; + uint32_t pacer_tuner_type; + union { + struct spdk_io_pacer_tuner *pacer_tuner; + struct spdk_io_pacer_tuner2 *pacer_tuner2; + }; }; struct spdk_io_pacer_tuner { @@ -127,22 +134,24 @@ io_pacer_poll(void *arg) #endif /* SPDK_CONFIG_VTUNE */ pacer->stat.calls++; - if (ticks_diff < pacer->period_ticks) { - return 0; + if (ticks_diff >= pacer->period_ticks){ + pacer->stat.total_ticks = cur_tick - pacer->first_tick; + pacer->last_tick = cur_tick - ticks_diff % pacer->period_ticks; + pacer->stat.polls++; + pacer->remaining_credit = spdk_min(pacer->remaining_credit + pacer->credit, + pacer->credit); + if (pacer->num_ios == 0) { + pacer->stat.no_ios++; + } } - pacer->stat.total_ticks = cur_tick - pacer->first_tick; - pacer->last_tick = cur_tick - ticks_diff % pacer->period_ticks; - pacer->stat.polls++; - - pacer->remaining_credit = spdk_min(pacer->remaining_credit + pacer->credit, - pacer->credit); - - if (pacer->num_ios == 0) { - pacer->stat.no_ios++; +#if SPDK_NVMF_RDMA_IO_PACER_CHECK_CREDITS_ONLY_AT_PACER_PERIOD + else { + return 0; } +#endif /* SPDK_NVMF_RDMA_IO_PACER_CHECK_CREDITS_ONLY_AT_PACER_PERIOD */ while ((pacer->num_ios > 0) && - (pacer->remaining_credit > 0) && + (is_credit_available(pacer) == true) && (attempts_cnt < pacer->num_queues)) { next_queue %= pacer->num_queues; attempts_cnt++; @@ -182,7 +191,8 @@ struct spdk_io_pacer * spdk_io_pacer_create(uint32_t period_ns, uint32_t credit, uint32_t disk_credit, - spdk_io_pacer_pop_cb pop_cb) + spdk_io_pacer_pop_cb pop_cb, + uint32_t io_pacer_tuner_type) { struct spdk_io_pacer *pacer; @@ -201,6 +211,7 @@ spdk_io_pacer_create(uint32_t period_ns, pacer->pop_cb = pop_cb; pacer->first_tick = spdk_get_ticks(); pacer->last_tick = spdk_get_ticks(); + pacer->stat.nos_credit_unavailable = 0; pacer->poller = SPDK_POLLER_REGISTER(io_pacer_poll, (void *)pacer, 0); if (!pacer->poller) { SPDK_ERRLOG("Failed to create poller for IO pacer\n"); @@ -208,14 +219,15 @@ spdk_io_pacer_create(uint32_t period_ns, return NULL; } - SPDK_NOTICELOG("Created IO pacer %p: period_ns %u, period_ticks %lu, max_queues %u, credit %ld, disk_credit %u, core %u\n", + SPDK_NOTICELOG("Created IO pacer %p: period_ns %u, period_ticks %lu, max_queues %u, credit %ld, disk_credit %u, core %u, Tuner type: %u\n", pacer, period_ns, pacer->period_ticks, pacer->max_queues, pacer->credit, pacer->disk_credit, - spdk_env_get_current_core()); + spdk_env_get_current_core(), + io_pacer_tuner_type); return pacer; } @@ -435,6 +447,8 @@ spdk_io_pacer_tuner_create(struct spdk_io_pacer *pacer, tuner->step_ns = step_ns; tuner->min_pacer_period_ticks = pacer->period_ticks; tuner->max_pacer_period_ticks = 2 * tuner->min_pacer_period_ticks; + pacer->pacer_tuner_type = SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01; + pacer->pacer_tuner = tuner; if (0 != period_us) { tuner->poller = SPDK_POLLER_REGISTER(io_pacer_tune, (void *)tuner, period_us); @@ -475,6 +489,7 @@ struct spdk_io_pacer_tuner2 { uint64_t min_pacer_period_ticks; uint64_t max_pacer_period_ticks; struct spdk_poller *poller; + uint64_t min_threshold_offset; }; static int @@ -492,14 +507,15 @@ io_pacer_tune2(void *arg) static __thread uint32_t log_counter = 0; /* Try to log once per second, else it would be too much log */ if (log_counter % (SPDK_SEC_TO_NSEC / tuner->period_ns) == 0) { - SPDK_NOTICELOG("IO pacer tuner %p: pacer %p, value %u, new period %lu ticks, min %lu, polls %u. ios %u\n", + SPDK_NOTICELOG("IO pacer tuner %p: pacer %p, value %u, new period %lu ticks, min %lu, polls %lu. ios %lu, no credit %lu", tuner, pacer, v, new_period_ticks, tuner->min_pacer_period_ticks, pacer->stat.polls, - pacer->stat.ios); + pacer->stat.ios, + pacer->stat.nos_credit_unavailable); } log_counter++; @@ -515,6 +531,7 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer, uint64_t factor) { struct spdk_io_pacer_tuner2 *tuner; + uint32_t min_threshold_margin; assert(pacer != NULL); @@ -531,7 +548,11 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer, tuner->factor = factor; tuner->min_pacer_period_ticks = pacer->period_ticks; tuner->max_pacer_period_ticks = 4 * tuner->min_pacer_period_ticks; + pacer->pacer_tuner_type = SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02; + pacer->pacer_tuner2 = tuner; + min_threshold_margin = 100 / SPDK_NVMF_RDMA_IO_PACER_DEFAULT_MARGIN_ABOVE_CREDIT; /* variable parameter */ + tuner->min_threshold_offset = (tuner->min_threshold / min_threshold_margin); if (0 != period_us) { tuner->poller = SPDK_POLLER_REGISTER(io_pacer_tune2, (void *)tuner, period_us); if (!tuner->poller) { @@ -541,12 +562,13 @@ spdk_io_pacer_tuner2_create(struct spdk_io_pacer *pacer, } } - SPDK_NOTICELOG("Created IO pacer tuner %p: pacer %p, period_ns %lu, threshold %u, factor %lu\n", + SPDK_NOTICELOG("Created IO pacer tuner %p: pacer %p, period_ns %lu, threshold %u, factor %lu, allowed offset %lu\n", tuner, pacer, tuner->period_ns, tuner->min_threshold, - tuner->factor); + tuner->factor, + tuner->min_threshold_offset); return tuner; } @@ -574,3 +596,28 @@ spdk_io_pacer_tuner2_sub(struct spdk_io_pacer_tuner2 *tuner, uint32_t value) assert(tuner != NULL); tuner->value -= value; } + +bool +is_credit_available(struct spdk_io_pacer *pacer){ +#if SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY + /* + * Check the bytes in fight within allowed offset above the minimum allowed + * threshold value + */ + if (pacer->pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02){ + struct spdk_io_pacer_tuner2 *pacer_tuner = pacer->pacer_tuner2; + if ((pacer->remaining_credit > 0)&& + (pacer_tuner->value <= pacer_tuner->min_threshold + pacer_tuner->min_threshold_offset)){ + return true; + } + } else if (pacer->remaining_credit > 0){ + return true; + } +#else /* SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY */ + else if (pacer->remaining_credit > 0){ + return true; + } +#endif /* SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY */ + pacer->stat.nos_credit_unavailable++; + return false; +} diff --git a/lib/nvmf/io_pacer.h b/lib/nvmf/io_pacer.h index d3ad21e1ebb..beaccb4f735 100644 --- a/lib/nvmf/io_pacer.h +++ b/lib/nvmf/io_pacer.h @@ -42,6 +42,19 @@ #include "spdk_internal/log.h" #include "spdk/nvmf.h" +/* Tuner types +*/ +#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01 01 +#define SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02 02 + +/* + * Configures credit availability to be calculated along with the + * bytes in flight within in the allowed margin offset + */ +#define SPDK_NVMF_RDMA_IO_PACER_CHECK_CREDITS_ONLY_AT_PACER_PERIOD 1 +#define SPDK_NVMF_RDMA_IO_PACER_ALLOW_WITHIN_CREDIT_ONLY 1 +#define SPDK_NVMF_RDMA_IO_PACER_DEFAULT_MARGIN_ABOVE_CREDIT 1 + struct spdk_io_pacer; struct spdk_io_pacer_tuner; struct spdk_io_pacer_tuner2; @@ -68,7 +81,8 @@ typedef void (*spdk_io_pacer_pop_cb)(void *io); struct spdk_io_pacer *spdk_io_pacer_create(uint32_t period_ns, uint32_t credit, uint32_t disk_credit, - spdk_io_pacer_pop_cb pop_cb); + spdk_io_pacer_pop_cb pop_cb, + uint32_t io_pacer_tuner_type); void spdk_io_pacer_destroy(struct spdk_io_pacer *pacer); int spdk_io_pacer_create_queue(struct spdk_io_pacer *pacer, uint64_t key); int spdk_io_pacer_destroy_queue(struct spdk_io_pacer *pacer, uint64_t key); @@ -89,6 +103,8 @@ void spdk_io_pacer_tuner2_destroy(struct spdk_io_pacer_tuner2 *tuner); void spdk_io_pacer_tuner2_add(struct spdk_io_pacer_tuner2 *tuner, uint32_t value); void spdk_io_pacer_tuner2_sub(struct spdk_io_pacer_tuner2 *tuner, uint32_t value); +bool is_credit_available(struct spdk_io_pacer *pacer); + static inline void drive_stats_lock(struct spdk_io_pacer_drives_stats *stats) { rte_spinlock_lock(&stats->lock); } diff --git a/lib/nvmf/rdma.c b/lib/nvmf/rdma.c index 0b8e03a6178..bba214fe2aa 100644 --- a/lib/nvmf/rdma.c +++ b/lib/nvmf/rdma.c @@ -2227,7 +2227,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, STAILQ_REMOVE_HEAD(&rgroup->group.pending_buf_queue, buf_link); - if (rgroup->pacer_tuner2 && (rtransport->transport.opts.io_pacer_tuner_type == 1)) { + if (rgroup->pacer_tuner2 && (rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02)) { spdk_io_pacer_tuner2_add(rgroup->pacer_tuner2, rdma_req->req.length); } @@ -2406,7 +2406,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, } if (rgroup->pacer_tuner2 && - (rtransport->transport.opts.io_pacer_tuner_type == 1)) { + (rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02)) { spdk_io_pacer_tuner2_sub(rgroup->pacer_tuner2, rdma_req->req.length); } @@ -2446,7 +2446,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, #define SPDK_NVMF_RDMA_DIF_INSERT_OR_STRIP false #define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_PERIOD 0 #define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_THRESHOLD 0 -#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_TYPE 1 /* Buffers based tuner */ +#define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_TYPE SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_02 /* Buffers based tuner */ #define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_PERIOD 10000 /* us */ #define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_STEP 1000 /* ns */ #define SPDK_NVMF_RDMA_DEFAULT_IO_PACER_TUNER_THRESHOLD 12*1024*1024 @@ -3561,7 +3561,8 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) rgroup->pacer = spdk_io_pacer_create(transport->opts.io_pacer_period, transport->opts.io_pacer_credit, transport->opts.io_pacer_disk_credit, - nvmf_rdma_io_pacer_pop_cb); + nvmf_rdma_io_pacer_pop_cb, + transport->opts.io_pacer_tuner_type); if (!rgroup->pacer) { SPDK_ERRLOG("Failed to create IO pacer\n"); spdk_nvmf_rdma_poll_group_destroy(&rgroup->group); @@ -3569,7 +3570,7 @@ spdk_nvmf_rdma_poll_group_create(struct spdk_nvmf_transport *transport) return NULL; } - if (transport->opts.io_pacer_tuner_type == 0) { + if (transport->opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01) { rgroup->pacer_tuner = spdk_io_pacer_tuner_create(rgroup->pacer, transport->opts.io_pacer_tuner_period, transport->opts.io_pacer_tuner_step); @@ -3718,7 +3719,7 @@ spdk_nvmf_rdma_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group) rtransport = SPDK_CONTAINEROF(rgroup->group.transport, struct spdk_nvmf_rdma_transport, transport); if (rgroup->pacer) { - if (rtransport->transport.opts.io_pacer_tuner_type == 0) { + if (rtransport->transport.opts.io_pacer_tuner_type == SPDK_NVMF_RDMA_IO_PACER_TUNER_TYPE_01) { spdk_io_pacer_tuner_destroy(rgroup->pacer_tuner); } else { spdk_io_pacer_tuner2_destroy(rgroup->pacer_tuner2);