diff --git a/src/k8s_etcd.rs b/src/k8s_etcd.rs index 259ec1e7..9165fefc 100644 --- a/src/k8s_etcd.rs +++ b/src/k8s_etcd.rs @@ -59,7 +59,19 @@ impl InMemoryK8sEtcd { let key = key.clone(); let etcd_client = Arc::clone(etcd_client); tokio::spawn(async move { - etcd_client.kv_client().delete(key.as_bytes(), None).await?; + loop { + let delete_response = etcd_client.kv_client().delete(key.as_bytes(), None).await; + + if is_too_many_requests_error(&delete_response) { + continue; + } + + match delete_response { + Ok(_) => break, + Err(_) => delete_response.context(format!("during etcd delete {}", key))?, + }; + } + anyhow::Ok(()) }) }) @@ -166,6 +178,16 @@ impl InMemoryK8sEtcd { } } +fn is_too_many_requests_error(delete_response: &std::prelude::v1::Result) -> bool { + match delete_response { + Ok(_) => false, + Err(err) => match err { + etcd_client::Error::GRpcStatus(status) => status.message() == "etcdserver: too many requests", + _ => false, + }, + } +} + pub(crate) async fn get_etcd_json(client: &InMemoryK8sEtcd, k8slocation: &K8sResourceLocation) -> Result> { let etcd_result = client .get(k8slocation.as_etcd_key())