prov/efa: Fall back to zero sl when non-zero sl qp creation failed

When driver doesn't support non-zero sl, the qp creation will fail. In this case, fallback to zero sl qp. Signed-off-by: Shi Jin <[email protected]>
pmodels · Nov 7, 2024 · 230b840 · 230b840
1 parent a03c5a7
commit 230b840
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -186,14 +186,25 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
 #endif
 		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;
 #if HAVE_EFADV_SL
+		efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL;
 		if (tclass == FI_TC_LOW_LATENCY)
-			efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL;
+			efa_attr.sl = EFA_QP_LOW_LATENCY_SERVICE_LEVEL;
 #endif
 		(*qp)->ibv_qp = efadv_create_qp_ex(
 			init_attr_ex->pd->context, init_attr_ex, &efa_attr,
 			sizeof(struct efadv_qp_init_attr));
 	}
 
+#if HAVE_EFADV_SL
+	if (!(*qp)->ibv_qp && tclass == FI_TC_LOW_LATENCY) {
+		EFA_INFO(FI_LOG_EP_CTRL, "ibv_create_qp failed with sl %u, errno: %d. Retrying with default sl.\n", efa_attr.sl, errno);
+		efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL;
+		(*qp)->ibv_qp = efadv_create_qp_ex(
+			init_attr_ex->pd->context, init_attr_ex, &efa_attr,
+			sizeof(struct efadv_qp_init_attr));
+	}
+#endif
+
 	if (!(*qp)->ibv_qp) {
 		EFA_WARN(FI_LOG_EP_CTRL, "ibv_create_qp failed. errno: %d\n", errno);
 		free(*qp);

diff --git a/prov/efa/src/efa_base_ep.h b/prov/efa/src/efa_base_ep.h
@@ -12,7 +12,8 @@
 #include "ofi_util.h"
 #include "rdm/efa_rdm_protocol.h"
 
-#define EFA_QP_DEFAULT_SERVICE_LEVEL 8
+#define EFA_QP_DEFAULT_SERVICE_LEVEL 0
+#define EFA_QP_LOW_LATENCY_SERVICE_LEVEL 8
 
 struct efa_qp {
 	struct ibv_qp *ibv_qp;