DAOS-15040 pool: Disconnect handles when stopping

Signed-off-by: Li Wei <[email protected]>
Required-githooks: true

src/include/daos_srv/pool.h

@@ -42,6 +42,7 @@
 struct ds_pool {
 	struct daos_llink	sp_entry;
 	uuid_t			sp_uuid;	/* pool UUID */
+	d_list_t		sp_hdls;
 	ABT_rwlock		sp_lock;
 	struct pool_map		*sp_map;
 	uint32_t		sp_map_version;	/* temporary */
@@ -120,6 +121,7 @@ void ds_pool_get(struct ds_pool *pool);
  */
 struct ds_pool_hdl {
 	d_list_t		sph_entry;
+	d_list_t		sph_pool_entry;
 	uuid_t			sph_uuid;	/* of the pool handle */
 	uint64_t		sph_flags;	/* user-provided flags */
 	uint64_t		sph_sec_capas;	/* access capabilities */

src/pool/srv_internal.h

@@ -210,6 +210,7 @@ void ds_pool_upgrade_handler(crt_rpc_t *rpc);
  */
 int ds_pool_cache_init(void);
 void ds_pool_cache_fini(void);
+int ds_pool_lookup_internal(const uuid_t uuid, struct ds_pool **pool);
 int ds_pool_hdl_hash_init(void);
 void ds_pool_hdl_hash_fini(void);
 void ds_pool_hdl_delete_all(void);

src/pool/srv_iv.c

@@ -974,7 +974,10 @@ pool_iv_ent_refresh(struct ds_iv_entry *entry, struct ds_iv_key *key,
 	struct ds_pool		*pool = 0;
 	int			rc;
 
-	rc = ds_pool_lookup(entry->ns->iv_pool_uuid, &pool);
+	if (src == NULL)
+		rc = ds_pool_lookup_internal(entry->ns->iv_pool_uuid, &pool);
+	else
+		rc = ds_pool_lookup(entry->ns->iv_pool_uuid, &pool);
 	if (rc) {
 		D_WARN("No pool "DF_UUID": %d\n", DP_UUID(entry->ns->iv_pool_uuid), rc);
 		if (rc == -DER_NONEXIST)

src/pool/srv_target.c

@@ -689,6 +689,7 @@ pool_alloc_ref(void *key, unsigned int ksize, void *varg,
 
 	D_INIT_LIST_HEAD(&pool->sp_ec_ephs_list);
 	uuid_copy(pool->sp_uuid, key);
+	D_INIT_LIST_HEAD(&pool->sp_hdls);
 	pool->sp_map_version = arg->pca_map_version;
 	pool->sp_reclaim = DAOS_RECLAIM_LAZY; /* default reclaim strategy */
 	pool->sp_policy_desc.policy =
@@ -765,6 +766,8 @@ pool_free_ref(struct daos_llink *llink)
 
 	D_DEBUG(DB_MGMT, DF_UUID": freeing\n", DP_UUID(pool->sp_uuid));
 
+	D_ASSERT(d_list_empty(&pool->sp_hdls));
+
 	rc = dss_thread_collective(pool_child_delete_one, pool->sp_uuid, 0);
 	if (rc == -DER_CANCELED)
 		D_DEBUG(DB_MD, DF_UUID": no ESs\n", DP_UUID(pool->sp_uuid));
@@ -835,13 +838,8 @@ ds_pool_cache_fini(void)
 	daos_lru_cache_destroy(pool_cache);
 }
 
-/**
- * If the pool can not be found due to non-existence or it is being stopped, then
- * @pool will be set to NULL and return proper failure code, otherwise return 0 and
- * set @pool.
- */
 int
-ds_pool_lookup(const uuid_t uuid, struct ds_pool **pool)
+ds_pool_lookup_internal(const uuid_t uuid, struct ds_pool **pool)
 {
 	struct daos_llink	*llink;
 	int			 rc;
@@ -855,6 +853,23 @@ ds_pool_lookup(const uuid_t uuid, struct ds_pool **pool)
 		return rc;
 
 	*pool = pool_obj(llink);
+	return 0;
+}
+
+/**
+ * If the pool can not be found due to non-existence or it is being stopped, then
+ * @pool will be set to NULL and return proper failure code, otherwise return 0 and
+ * set @pool.
+ */
+int
+ds_pool_lookup(const uuid_t uuid, struct ds_pool **pool)
+{
+	int rc;
+
+	rc = ds_pool_lookup_internal(uuid, pool);
+	if (rc != 0)
+		return rc;
+
 	if ((*pool)->sp_stopping) {
 		D_DEBUG(DB_MD, DF_UUID": is in stopping\n", DP_UUID(uuid));
 		ds_pool_put(*pool);
@@ -1066,6 +1081,17 @@ ds_pool_start(uuid_t uuid)
 	return rc;
 }
 
+static void pool_tgt_disconnect(struct ds_pool_hdl *hdl);
+
+static void
+pool_tgt_disconnect_all(struct ds_pool *pool)
+{
+	struct ds_pool_hdl *hdl;
+
+	while ((hdl = d_list_pop_entry(&pool->sp_hdls, struct ds_pool_hdl, sph_pool_entry)) != NULL)
+		pool_tgt_disconnect(hdl);
+}
+
 /*
  * Stop a pool. Must be called on the system xstream. Release the ds_pool
  * object reference held by ds_pool_start. Only for mgmt and pool modules.
@@ -1083,6 +1109,8 @@ ds_pool_stop(uuid_t uuid)
 	D_ASSERT(!pool->sp_stopping);
 	pool->sp_stopping = 1;
 
+	pool_tgt_disconnect_all(pool);
+
 	ds_iv_ns_stop(pool->sp_iv_ns);
 	ds_pool_tgt_ec_eph_query_abort(pool);
 	pool_fetch_hdls_ult_abort(pool);
@@ -1155,6 +1183,7 @@ pool_hdl_rec_free(struct d_hash_table *htable, d_list_t *rlink)
 	D_DEBUG(DB_MD, DF_UUID": freeing "DF_UUID"\n",
 		DP_UUID(hdl->sph_pool->sp_uuid), DP_UUID(hdl->sph_uuid));
 	D_ASSERT(d_hash_rec_unlinked(&hdl->sph_entry));
+	D_ASSERT(d_list_empty(&hdl->sph_pool_entry));
 	D_ASSERTF(hdl->sph_ref == 0, "%d\n", hdl->sph_ref);
 	daos_iov_free(&hdl->sph_cred);
 	ds_pool_put(hdl->sph_pool);
@@ -1492,8 +1521,18 @@ ds_pool_tgt_connect(struct ds_pool *pool, struct pool_iv_conn *pic)
 		D_GOTO(out, rc);
 	}
 
+	if (pool->sp_stopping) {
+		daos_iov_free(&hdl->sph_cred);
+		ds_pool_put(pool);
+		rc = -DER_SHUTDOWN;
+		goto out;
+	}
+
+	d_list_add(&hdl->sph_pool_entry, &hdl->sph_pool->sp_hdls);
+
 	rc = pool_hdl_add(hdl);
 	if (rc != 0) {
+		d_list_del_init(&hdl->sph_pool_entry);
 		daos_iov_free(&hdl->sph_cred);
 		ds_pool_put(pool);
 		D_GOTO(out, rc);
@@ -1508,6 +1547,16 @@ ds_pool_tgt_connect(struct ds_pool *pool, struct pool_iv_conn *pic)
 	return rc;
 }
 
+static void
+pool_tgt_disconnect(struct ds_pool_hdl *hdl)
+{
+	D_DEBUG(DB_MD, DF_UUID ": hdl=" DF_UUID "\n", DP_UUID(hdl->sph_pool->sp_uuid),
+		DP_UUID(hdl->sph_uuid));
+	ds_pool_iv_conn_hdl_invalidate(hdl->sph_pool, hdl->sph_uuid);
+	pool_hdl_delete(hdl);
+	d_list_del_init(&hdl->sph_pool_entry);
+}
+
 void
 ds_pool_tgt_disconnect(uuid_t uuid)
 {
@@ -1520,9 +1569,8 @@ ds_pool_tgt_disconnect(uuid_t uuid)
 		return;
 	}
 
-	ds_pool_iv_conn_hdl_invalidate(hdl->sph_pool, uuid);
+	pool_tgt_disconnect(hdl);
 
-	pool_hdl_delete(hdl);
 	ds_pool_hdl_put(hdl);
 }