diff --git a/configs/config.dev.yaml b/configs/config.dev.yaml index 10c4c408..8516312f 100644 --- a/configs/config.dev.yaml +++ b/configs/config.dev.yaml @@ -135,6 +135,23 @@ sender: ## slaves to be cycled through is_master: True + # default_rate_def: + # # how many messages in a period before they are dropped + # hard_limit: 30 + # # how many messages in a period before notification is sent out + # soft_limit: 20 + # # period length for hard limit + # hard_duration: 1 + # # period length for soft limit + # soft_duration: 1 + # # time to wait before messages can be created again + # wait_time: 3600 + # # target that will receive notification for soft quota breaches + # target_name: foo_team + # target_role: team + # # plan for incident that will be raised if there is a hard quota breach + # plan_name: bar_plan + #slaves: # - host: 127.0.0.1 # port: 2322 diff --git a/src/iris/bin/sender.py b/src/iris/bin/sender.py index 007ba26f..fb6565d4 100644 --- a/src/iris/bin/sender.py +++ b/src/iris/bin/sender.py @@ -1695,7 +1695,7 @@ def init_sender(config): }] global quota - quota = ApplicationQuota(db, cache.targets_for_role, message_send_enqueue, config['sender'].get('sender_app')) + quota = ApplicationQuota(db, cache.targets_for_role, message_send_enqueue, config['sender'].get('sender_app'), config['sender'].get('default_rate_def', {})) global coordinator zk_hosts = config['sender'].get('zookeeper_cluster', False) diff --git a/src/iris/sender/quota.py b/src/iris/sender/quota.py index e0a304f0..5421efbb 100644 --- a/src/iris/sender/quota.py +++ b/src/iris/sender/quota.py @@ -59,8 +59,22 @@ class ApplicationQuota(object): - def __init__(self, db, expand_targets, message_send_enqueue, sender_app): + def __init__(self, db, expand_targets, message_send_enqueue, sender_app, rate_configs): self.db = db + # configure default rate limiting params for messages + hard_limit = rate_configs.get('hard_limit', 6000) + soft_limit = rate_configs.get('soft_limit', 600) + hard_duration = rate_configs.get('hard_duration', 1) + soft_duration = rate_configs.get('soft_duration', 1) + wait_time = rate_configs.get('wait_time', 3600) + plan_name = rate_configs.get('plan_name', None) + target_name = rate_configs.get('target_name', None) + target_role = rate_configs.get('target_role', None) + target = None + if target_name and target_role: + target = (target_name, target_role) + self.default_rate_def = (hard_limit, soft_limit, hard_duration, soft_duration, wait_time, plan_name, target) + self.expand_targets = expand_targets self.message_send_enqueue = message_send_enqueue self.iris_application = None @@ -95,6 +109,11 @@ def refresh(self): for application, hard_limit, soft_limit, hard_duration, soft_duration, target_name, target_role, plan_name, wait_time in self.get_new_rules(): new_rates[application] = (hard_limit, soft_limit, hard_duration // 60, soft_duration // 60, wait_time, plan_name, (target_name, target_role)) + # for any application that does not have a quota defined set it to a default value + for app in iris.cache.applications: + if not new_rates.get(app): + new_rates[app] = self.default_rate_def + old_keys = self.rates.keys() new_keys = new_rates.keys() @@ -163,8 +182,9 @@ def allow_send(self, message): if hard_quota_usage > hard_limit: metrics.incr('quota_hard_exceed_cnt') - with self.last_incidents_mutex: - self.notify_incident(application, hard_limit, len(hard_buckets), plan_name, wait_time) + if plan_name: + with self.last_incidents_mutex: + self.notify_incident(application, hard_limit, len(hard_buckets), plan_name, wait_time) return False # If soft limit breached, just notify owner and still send @@ -177,8 +197,9 @@ def allow_send(self, message): if soft_quota_usage > soft_limit: metrics.incr('quota_soft_exceed_cnt') - with self.last_soft_quota_notification_time_mutex: - self.notify_target(application, soft_limit, len(soft_buckets), *target) + if target: + with self.last_soft_quota_notification_time_mutex: + self.notify_target(application, soft_limit, len(soft_buckets), *target) return True return True diff --git a/test/test_sender.py b/test/test_sender.py index 556e9abe..55605ee7 100644 --- a/test/test_sender.py +++ b/test/test_sender.py @@ -348,7 +348,7 @@ def test_quotas(mocker): )]) mocker.patch('iris.sender.quota.ApplicationQuota.notify_incident') mocker.patch('iris.sender.quota.ApplicationQuota.notify_target') - quotas = ApplicationQuota(None, None, None, None) + quotas = ApplicationQuota(None, None, None, None, {}) sleep(1) # ensure drop messages don't count