From 611a9e7a04a159c45d0ca75e0300982f9575eb76 Mon Sep 17 00:00:00 2001 From: xuruidong Date: Wed, 18 Dec 2024 14:12:05 +0800 Subject: [PATCH 1/7] perf: accelerate the creation of the consumer cache Signed-off-by: xuruidong --- apisix/consumer.lua | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index 0eb55b5e0994..f8a388543894 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -80,7 +80,10 @@ local function filter_consumers_list(data_list) return list end -local function plugin_consumer() +local plugin_consumer +do + local consumers_id_cache = {} +function plugin_consumer() local plugins = {} if consumers.values == nil then @@ -101,10 +104,19 @@ local function plugin_consumer() if not plugins[name] then plugins[name] = { nodes = {}, + len = 0, conf_version = consumers.conf_version } end + local cached_consumer = consumers_id_cache[val.value.id] + if cached_consumer and + cached_consumer.modifiedIndex == val.modifiedIndex then + plugins[name].len = plugins[name].len + 1 + core.table.insert(plugins[name].nodes, plugins[name].len, + cached_consumer) + goto CONTINUE_INTERNAL + end -- if the val is a Consumer, clone it to the local consumer; -- if the val is a Credential, to get the Consumer by consumer_name and then clone -- it to the local consumer. @@ -141,7 +153,10 @@ local function plugin_consumer() consumer.auth_conf = config core.log.info("consumer:", core.json.delay_encode(consumer)) core.table.insert(plugins[name].nodes, consumer) + consumers_id_cache[val.value.id] = consumer end + + ::CONTINUE_INTERNAL:: end ::CONTINUE:: @@ -150,6 +165,9 @@ local function plugin_consumer() return plugins end +end + + _M.filter_consumers_list = filter_consumers_list function _M.get_consumer_key_from_credential_key(key) @@ -186,15 +204,23 @@ function _M.consumers() end -local function create_consume_cache(consumers_conf, key_attr) +local consumers_plugin_key_cache = {} +local function create_consume_cache(consumers_conf, key_attr, cache) local consumer_names = {} for _, consumer in ipairs(consumers_conf.nodes) do core.log.info("consumer node: ", core.json.delay_encode(consumer)) - local new_consumer = core.table.clone(consumer) - new_consumer.auth_conf = secret.fetch_secrets(new_consumer.auth_conf, true, - new_consumer.auth_conf, "") - consumer_names[new_consumer.auth_conf[key_attr]] = new_consumer + if cache and cache[consumer.auth_conf[key_attr]] and + cache[consumer.auth_conf[key_attr]].modifiedIndex == + consumer.modifiedIndex then + consumer_names[consumer.auth_conf[key_attr]] = + cache[consumer.auth_conf[key_attr]] + else + local new_consumer = core.table.clone(consumer) + new_consumer.auth_conf = secret.fetch_secrets(new_consumer.auth_conf, true, + new_consumer.auth_conf, "") + consumer_names[new_consumer.auth_conf[key_attr]] = new_consumer + end end return consumer_names @@ -203,8 +229,10 @@ end function _M.consumers_kv(plugin_name, consumer_conf, key_attr) local consumers = lrucache("consumers_key#" .. plugin_name, consumer_conf.conf_version, - create_consume_cache, consumer_conf, key_attr) + create_consume_cache, consumer_conf, key_attr, + consumers_plugin_key_cache[plugin_name]) + consumers_plugin_key_cache[plugin_name] = consumers return consumers end From 628350ba836a7ed60886333f7c8ed89423e77492 Mon Sep 17 00:00:00 2001 From: xuruidong Date: Mon, 23 Dec 2024 19:50:05 +0800 Subject: [PATCH 2/7] refactor: use lrucache Signed-off-by: xuruidong --- apisix/consumer.lua | 45 +++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index f8a388543894..5ccd74df2d4c 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -82,7 +82,7 @@ end local plugin_consumer do - local consumers_id_cache = {} + local consumers_id_cache = {} function plugin_consumer() local plugins = {} @@ -110,7 +110,7 @@ function plugin_consumer() end local cached_consumer = consumers_id_cache[val.value.id] - if cached_consumer and + if cached_consumer and cached_consumer.modifiedIndex == val.modifiedIndex then plugins[name].len = plugins[name].len + 1 core.table.insert(plugins[name].nodes, plugins[name].len, @@ -204,35 +204,44 @@ function _M.consumers() end -local consumers_plugin_key_cache = {} -local function create_consume_cache(consumers_conf, key_attr, cache) +local create_consume_cache +do + local consumers_plugin_key_lrucache_tab = {} + +local function create_new_consumer(consumer) + local new_consumer = core.table.clone(consumer) + new_consumer.auth_conf = secret.fetch_secrets(new_consumer.auth_conf, false) + return new_consumer +end + + +function create_consume_cache(consumers_conf, key_attr, plugin_name) local consumer_names = {} + local lru_cache = consumers_plugin_key_lrucache_tab[plugin_name] + if lru_cache == nil then + lru_cache = core.lrucache.new({ + ttl = 60 * 60 * 24, count = 20480 + }) + consumers_plugin_key_lrucache_tab[plugin_name] = lru_cache + end for _, consumer in ipairs(consumers_conf.nodes) do core.log.info("consumer node: ", core.json.delay_encode(consumer)) - if cache and cache[consumer.auth_conf[key_attr]] and - cache[consumer.auth_conf[key_attr]].modifiedIndex == - consumer.modifiedIndex then - consumer_names[consumer.auth_conf[key_attr]] = - cache[consumer.auth_conf[key_attr]] - else - local new_consumer = core.table.clone(consumer) - new_consumer.auth_conf = secret.fetch_secrets(new_consumer.auth_conf, true, - new_consumer.auth_conf, "") - consumer_names[new_consumer.auth_conf[key_attr]] = new_consumer - end + local new_consumer = lru_cache(consumer.auth_conf[key_attr], + consumer.modifiedIndex, create_new_consumer, consumer) + consumer_names[consumer.auth_conf[key_attr]] = new_consumer end return consumer_names end +end + function _M.consumers_kv(plugin_name, consumer_conf, key_attr) local consumers = lrucache("consumers_key#" .. plugin_name, consumer_conf.conf_version, - create_consume_cache, consumer_conf, key_attr, - consumers_plugin_key_cache[plugin_name]) + create_consume_cache, consumer_conf, key_attr, plugin_name) - consumers_plugin_key_cache[plugin_name] = consumers return consumers end From f2f88a07da599c53aee76617a616aaa0cf69447f Mon Sep 17 00:00:00 2001 From: xuruidong Date: Fri, 27 Dec 2024 18:19:27 +0800 Subject: [PATCH 3/7] refactor: replace consumer_id_cache with lrucache Signed-off-by: xuruidong --- apisix/consumer.lua | 122 +++++++++++++++++++++++--------------------- 1 file changed, 64 insertions(+), 58 deletions(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index 5ccd74df2d4c..92a243ab4cfc 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -36,6 +36,11 @@ local lrucache = core.lrucache.new({ ttl = 300, count = 512 }) +-- Please calculate and set the value of the "consumers_count_for_lrucache" +-- variable based on the number of consumers in the current environment, +-- taking into account the appropriate adjustment coefficient. +local consumers_count_for_lrucache = 4096 + local function remove_etcd_prefix(key) local prefix = "" local local_conf = config_local.local_conf() @@ -82,7 +87,50 @@ end local plugin_consumer do - local consumers_id_cache = {} + local consumers_id_lrucache = core.lrucache.new({ + count = consumers_count_for_lrucache + }) + +local function construct_consumer_data(val, plugin_config) + -- if the val is a Consumer, clone it to the local consumer; + -- if the val is a Credential, to get the Consumer by consumer_name and then clone + -- it to the local consumer. + local consumer + if is_credential_etcd_key(val.key) then + local consumer_name = get_consumer_name_from_credential_etcd_key(val.key) + local the_consumer = consumers:get(consumer_name) + if the_consumer and the_consumer.value then + consumer = core.table.clone(the_consumer.value) + consumer.modifiedIndex = the_consumer.modifiedIndex + consumer.credential_id = get_credential_id_from_etcd_key(val.key) + else + -- Normally wouldn't get here: + -- it should belong to a consumer for any credential. + core.log.error("failed to get the consumer for the credential,", + " a wild credential has appeared!", + " credential key: ", val.key, ", consumer name: ", consumer_name) + return nil, "failed to get the consumer for the credential" + end + else + consumer = core.table.clone(val.value) + consumer.modifiedIndex = val.modifiedIndex + end + + -- if the consumer has labels, set the field custom_id to it. + -- the custom_id is used to set in the request headers to the upstream. + if consumer.labels then + consumer.custom_id = consumer.labels["custom_id"] + end + + -- Note: the id here is the key of consumer data, which + -- is 'username' field in admin + consumer.consumer_name = consumer.id + consumer.auth_conf = plugin_config + + return consumer +end + + function plugin_consumer() local plugins = {} @@ -109,54 +157,17 @@ function plugin_consumer() } end - local cached_consumer = consumers_id_cache[val.value.id] - if cached_consumer and - cached_consumer.modifiedIndex == val.modifiedIndex then - plugins[name].len = plugins[name].len + 1 - core.table.insert(plugins[name].nodes, plugins[name].len, - cached_consumer) - goto CONTINUE_INTERNAL - end - -- if the val is a Consumer, clone it to the local consumer; - -- if the val is a Credential, to get the Consumer by consumer_name and then clone - -- it to the local consumer. - local consumer - if is_credential_etcd_key(val.key) then - local consumer_name = get_consumer_name_from_credential_etcd_key(val.key) - local the_consumer = consumers:get(consumer_name) - if the_consumer and the_consumer.value then - consumer = core.table.clone(the_consumer.value) - consumer.modifiedIndex = the_consumer.modifiedIndex - consumer.credential_id = get_credential_id_from_etcd_key(val.key) - else - -- Normally wouldn't get here: - -- it should belong to a consumer for any credential. - core.log.error("failed to get the consumer for the credential,", - " a wild credential has appeared!", - " credential key: ", val.key, ", consumer name: ", consumer_name) - goto CONTINUE - end - else - consumer = core.table.clone(val.value) - consumer.modifiedIndex = val.modifiedIndex - end - - -- if the consumer has labels, set the field custom_id to it. - -- the custom_id is used to set in the request headers to the upstream. - if consumer.labels then - consumer.custom_id = consumer.labels["custom_id"] + local consumer, err = consumers_id_lrucache(val.value.id, + val.modifiedIndex, construct_consumer_data, val, config) + if consumer == nil then + goto CONTINUE end - -- Note: the id here is the key of consumer data, which - -- is 'username' field in admin - consumer.consumer_name = consumer.id - consumer.auth_conf = config + plugins[name].len = plugins[name].len + 1 + core.table.insert(plugins[name].nodes, plugins[name].len, + consumer) core.log.info("consumer:", core.json.delay_encode(consumer)) - core.table.insert(plugins[name].nodes, consumer) - consumers_id_cache[val.value.id] = consumer end - - ::CONTINUE_INTERNAL:: end ::CONTINUE:: @@ -206,29 +217,24 @@ end local create_consume_cache do - local consumers_plugin_key_lrucache_tab = {} + local consumer_lrucache = core.lrucache.new({ + count = consumers_count_for_lrucache + }) -local function create_new_consumer(consumer) +local function fill_consumer_secret(consumer) local new_consumer = core.table.clone(consumer) new_consumer.auth_conf = secret.fetch_secrets(new_consumer.auth_conf, false) return new_consumer end -function create_consume_cache(consumers_conf, key_attr, plugin_name) +function create_consume_cache(consumers_conf, key_attr) local consumer_names = {} - local lru_cache = consumers_plugin_key_lrucache_tab[plugin_name] - if lru_cache == nil then - lru_cache = core.lrucache.new({ - ttl = 60 * 60 * 24, count = 20480 - }) - consumers_plugin_key_lrucache_tab[plugin_name] = lru_cache - end for _, consumer in ipairs(consumers_conf.nodes) do core.log.info("consumer node: ", core.json.delay_encode(consumer)) - local new_consumer = lru_cache(consumer.auth_conf[key_attr], - consumer.modifiedIndex, create_new_consumer, consumer) + local new_consumer = consumer_lrucache(consumer, nil, + fill_consumer_secret, consumer) consumer_names[consumer.auth_conf[key_attr]] = new_consumer end @@ -240,7 +246,7 @@ end function _M.consumers_kv(plugin_name, consumer_conf, key_attr) local consumers = lrucache("consumers_key#" .. plugin_name, consumer_conf.conf_version, - create_consume_cache, consumer_conf, key_attr, plugin_name) + create_consume_cache, consumer_conf, key_attr) return consumers end From f7a1a231702bb919c1ea17ba8f4e45316754fded Mon Sep 17 00:00:00 2001 From: xuruidong Date: Sun, 29 Dec 2024 23:07:28 +0800 Subject: [PATCH 4/7] fix: multi-auth --- apisix/consumer.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index 92a243ab4cfc..d21cbef2ecaa 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -157,7 +157,7 @@ function plugin_consumer() } end - local consumer, err = consumers_id_lrucache(val.value.id, + local consumer = consumers_id_lrucache(val.value.id .. name, val.modifiedIndex, construct_consumer_data, val, config) if consumer == nil then goto CONTINUE From a5d1d13d47a46a480979525c0426629449ac80dc Mon Sep 17 00:00:00 2001 From: xuruidong Date: Fri, 3 Jan 2025 22:52:12 +0800 Subject: [PATCH 5/7] refactor: remove deepcopy --- apisix/consumer.lua | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index d21cbef2ecaa..cef2b48b6ba1 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -222,9 +222,8 @@ do }) local function fill_consumer_secret(consumer) - local new_consumer = core.table.clone(consumer) - new_consumer.auth_conf = secret.fetch_secrets(new_consumer.auth_conf, false) - return new_consumer + consumer.auth_conf = secret.fetch_secrets(consumer.auth_conf, false) + return consumer end From abdb1dbe1ab6e9bcd5deb70ad7958825e23980c4 Mon Sep 17 00:00:00 2001 From: xuruidong Date: Fri, 24 Jan 2025 14:49:03 +0800 Subject: [PATCH 6/7] Revert "refactor: remove deepcopy" This reverts commit a5d1d13d47a46a480979525c0426629449ac80dc. --- apisix/consumer.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index cef2b48b6ba1..d21cbef2ecaa 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -222,8 +222,9 @@ do }) local function fill_consumer_secret(consumer) - consumer.auth_conf = secret.fetch_secrets(consumer.auth_conf, false) - return consumer + local new_consumer = core.table.clone(consumer) + new_consumer.auth_conf = secret.fetch_secrets(new_consumer.auth_conf, false) + return new_consumer end From c4f576e3178804974de386de15a46cc64ccb1688 Mon Sep 17 00:00:00 2001 From: xuruidong Date: Sun, 26 Jan 2025 17:57:03 +0800 Subject: [PATCH 7/7] fix: consumer key should be obtained from the escaped consumer --- apisix/consumer.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/consumer.lua b/apisix/consumer.lua index f1ffae00c115..0ec39d7190ee 100644 --- a/apisix/consumer.lua +++ b/apisix/consumer.lua @@ -239,7 +239,7 @@ function create_consume_cache(consumers_conf, key_attr) core.log.info("consumer node: ", core.json.delay_encode(consumer)) local new_consumer = consumer_lrucache(consumer, nil, fill_consumer_secret, consumer) - consumer_names[consumer.auth_conf[key_attr]] = new_consumer + consumer_names[new_consumer.auth_conf[key_attr]] = new_consumer end return consumer_names