From b7086e179c9be1a4620e60e5fa410a8450536063 Mon Sep 17 00:00:00 2001 From: Kent Dong Date: Fri, 10 Jan 2025 10:26:42 +0800 Subject: [PATCH] Fix several known bugs related to AI Gateway (#121) * rel: Upgrade to Higress 2.0.5 * feat: Update ai-gateway configuration scripts * Update higress core versions to 2.0.6-rc.1 * Upgrade apiserver to 0.0.19 * Update pilot tag to 743ae53ed47f50e9b1f7acc3fdd8ced8068cf3ee --- all-in-one/Dockerfile | 9 +- all-in-one/get-ai-gateway.sh | 20 +-- .../scripts/config-template/ai-gateway.sh | 132 ++++++++++-------- compose/.env | 10 +- compose/docker-compose.yml | 2 +- 5 files changed, 95 insertions(+), 78 deletions(-) diff --git a/all-in-one/Dockerfile b/all-in-one/Dockerfile index e320a64..f2a0246 100644 --- a/all-in-one/Dockerfile +++ b/all-in-one/Dockerfile @@ -1,8 +1,8 @@ ARG HUB=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress ARG BASE_VERSION=2022-10-27T19-02-22 -ARG CORE_VERSION=2.0.4 -ARG CONSOLE_VERSION=1.4.6 -ARG APISERVER_VERSION=0.0.16 +ARG CORE_VERSION=2.0.6-rc.1 +ARG CONSOLE_VERSION=2.0.0 +ARG APISERVER_VERSION=0.0.19 ARG PROMETHEUS_VERSION=v2.40.7 ARG PROMTAIL_VERSION=2.9.4 ARG LOKI_VERSION=2.9.4 @@ -10,7 +10,8 @@ ARG GRAFANA_VERSION=9.3.6 FROM ${HUB}/api-server:${APISERVER_VERSION} AS apiserver FROM ${HUB}/higress:${CORE_VERSION} AS controller -FROM ${HUB}/pilot:${CORE_VERSION} AS pilot +# FROM ${HUB}/pilot:${CORE_VERSION} AS pilot +FROM ${HUB}/pilot:743ae53ed47f50e9b1f7acc3fdd8ced8068cf3ee AS pilot FROM ${HUB}/gateway:${CORE_VERSION} AS gateway FROM ${HUB}/console:${CONSOLE_VERSION} AS console FROM ${HUB}/prometheus:${PROMETHEUS_VERSION} AS prometheus diff --git a/all-in-one/get-ai-gateway.sh b/all-in-one/get-ai-gateway.sh index 8c351b7..c342ac9 100755 --- a/all-in-one/get-ai-gateway.sh +++ b/all-in-one/get-ai-gateway.sh @@ -146,15 +146,15 @@ runConfigWizard() { "OpenAI|OPENAI" "Aliyun Dashscope|DASHSCOPE" "Moonshot|MOONSHOT" - "Azure OpenAI|AZURE|configureAzureProvider" - "360 Zhinao|AI360" - "Github Models|GITHUB" - "Groq|GROQ" + # "Azure OpenAI|AZURE|configureAzureProvider" + # "360 Zhinao|AI360" + # "Github Models|GITHUB" + # "Groq|GROQ" "Baichuan AI|BAICHUAN" "01.AI|YI" - "DeepSeek|DEEPSEEK" + # "DeepSeek|DEEPSEEK" "Zhipu AI|ZHIPUAI" - "Ollama|OLLAMA|configureOllamaProvider" + # "Ollama|OLLAMA|configureOllamaProvider" "Claude|CLAUDE|configureClaudeProvider" # "Baidu AI Cloud|BAIDU" # "Tencent Hunyuan|HUNYUAN" @@ -162,12 +162,12 @@ runConfigWizard() { "Minimax|MINIMAX|configureMinimaxProvider" # "Cloudflare Workers AI|CLOUDFLARE" # "iFlyTek Spark|SPARK" - "Google Gemini|GEMINI" + # "Google Gemini|GEMINI" # "DeepL|DEEPL" - "Mistral AI|MISTRAL" - "Cohere|COHERE" + # "Mistral AI|MISTRAL" + # "Cohere|COHERE" "Doubao|DOUBAO" - "Coze|COZE" + # "Coze|COZE" ) local selectedIndex='' diff --git a/all-in-one/scripts/config-template/ai-gateway.sh b/all-in-one/scripts/config-template/ai-gateway.sh index 37f5308..9982ab9 100644 --- a/all-in-one/scripts/config-template/ai-gateway.sh +++ b/all-in-one/scripts/config-template/ai-gateway.sh @@ -5,43 +5,43 @@ ROOT=$(pwd) cd - >/dev/null source $ROOT/../base.sh -AI_PROXY_VERSION=${AI_PROXY_VERSION:-latest} -MODEL_ROUTER_VERSION=${MODEL_ROUTER_VERSION:-latest} +AI_PROXY_VERSION=${AI_PROXY_VERSION:-1.0.0} +MODEL_ROUTER_VERSION=${MODEL_ROUTER_VERSION:-1.0.0} function initializeLlmProviderConfigs() { local EXTRA_CONFIGS=() - initializeLlmProviderConfig aliyun qwen DASHSCOPE dashscope.aliyuncs.com - initializeLlmProviderConfig moonshot moonshot MOONSHOT api.moonshot.cn - initializeLlmProviderConfig openai openai OPENAI api.openai.com - initializeLlmProviderConfig ai360 ai360 AI360 api.360.cn - initializeLlmProviderConfig github github GITHUB models.inference.ai.azure.com - initializeLlmProviderConfig groq groq GROQ api.groq.com - initializeLlmProviderConfig baichuan baichuan BAICHUAN api.baichuan-ai.com - initializeLlmProviderConfig yi yi YI api.lingyiwanwu.com - initializeLlmProviderConfig deepseek deepseek DEEPSEEK api.deepseek.com - initializeLlmProviderConfig zhipuai zhipuai ZHIPUAI open.bigmodel.cn - # initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com - # initializeLlmProviderConfig hunyuan hunyuan HUNYUAN hunyuan.tencentcloudapi.com 443 "https" "" "${EXTRA_CONFIGS[@]}" - initializeLlmProviderConfig stepfun stepfun STEPFUN api.stepfun.com - # initializeLlmProviderConfig cloudflare cloudflare CLOUDFLARE api.cloudflare.com 443 "https" "" "${EXTRA_CONFIGS[@]}" - # initializeLlmProviderConfig spark spark SPARK spark-api-open.xf-yun.com 443 "https" "" "${EXTRA_CONFIGS[@]}" - initializeLlmProviderConfig gemini gemini GEMINI generativelanguage.googleapis.com - # initializeLlmProviderConfig deepl deepl DEEPL 443 "https" "" "${EXTRA_CONFIGS[@]}" - initializeLlmProviderConfig mistral mistral MISTRAL api.mistral.ai - initializeLlmProviderConfig cohere cohere COHERE api.cohere.com - initializeLlmProviderConfig doubao doubao DOUBAO ark.cn-beijing.volces.com - initializeLlmProviderConfig coze coze COZE api.coze.cn - - if [ -z "$AZURE_SERVICE_URL" ]; then - AZURE_SERVICE_URL="https://YOUR_RESOURCE_NAME.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT_NAME/chat/completions?api-version=2024-06-01" - fi - extractHostFromUrl "$AZURE_SERVICE_URL" - local AZURE_SERVICE_DOMAIN="$HOST" - EXTRA_CONFIGS=( - "azureServiceUrl=$AZURE_SERVICE_URL" - ) - initializeLlmProviderConfig azure azure AZURE "$AZURE_SERVICE_DOMAIN" "443" "https" "" "${EXTRA_CONFIGS[@]}" + initializeLlmProviderConfig aliyun qwen DASHSCOPE dashscope.aliyuncs.com "443" "https" "" "PRE" 'qwen-' + initializeLlmProviderConfig moonshot moonshot MOONSHOT api.moonshot.cn "443" "https" "" "PRE" 'moonshot-' + initializeLlmProviderConfig openai openai OPENAI api.openai.com "443" "https" "" "REGULAR" 'gpt\\-.*|o1\\-.*' + # initializeLlmProviderConfig ai360 ai360 AI360 api.360.cn "443" "https" "" "PRE" "" + # initializeLlmProviderConfig github github GITHUB models.inference.ai.azure.com "443" "https" "" "PRE" "" + # initializeLlmProviderConfig groq groq GROQ api.groq.com "443" "https" "" "PRE" "" + initializeLlmProviderConfig baichuan baichuan BAICHUAN api.baichuan-ai.com "443" "https" "" "PRE" "Baichuan" + initializeLlmProviderConfig yi yi YI api.lingyiwanwu.com "443" "https" "" "PRE" 'yi-' + initializeLlmProviderConfig deepseek deepseek DEEPSEEK api.deepseek.com "443" "https" "" "PRE" "deepseek" + initializeLlmProviderConfig zhipuai zhipuai ZHIPUAI open.bigmodel.cn "443" "https" "" "PRE" "GLM-" + # initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com "443" "https" "" "PRE" "ERNIE-" + # initializeLlmProviderConfig hunyuan hunyuan HUNYUAN hunyuan.tencentcloudapi.com "443" "https" "" "PRE" "hunyuan-" "${EXTRA_CONFIGS[@]}" + initializeLlmProviderConfig stepfun stepfun STEPFUN api.stepfun.com "443" "https" "" "PRE" "step-" + # initializeLlmProviderConfig cloudflare cloudflare CLOUDFLARE api.cloudflare.com "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}" + # initializeLlmProviderConfig spark spark SPARK spark-api-open.xf-yun.com "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}" + # initializeLlmProviderConfig gemini gemini GEMINI generativelanguage.googleapis.com "443" "https" "" "PRE" "gemini-" + # initializeLlmProviderConfig deepl deepl DEEPL "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}" + # initializeLlmProviderConfig mistral mistral MISTRAL api.mistral.ai "443" "https" "" "PRE" "" + # initializeLlmProviderConfig cohere cohere COHERE api.cohere.com "443" "https" "" "PRE" "" + initializeLlmProviderConfig doubao doubao DOUBAO ark.cn-beijing.volces.com "443" "https" "" "PRE" "doubao-" + # initializeLlmProviderConfig coze coze COZE api.coze.cn "443" "https" "" "PRE" "" + + # if [ -z "$AZURE_SERVICE_URL" ]; then + # AZURE_SERVICE_URL="https://YOUR_RESOURCE_NAME.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT_NAME/chat/completions?api-version=2024-06-01" + # fi + # extractHostFromUrl "$AZURE_SERVICE_URL" + # local AZURE_SERVICE_DOMAIN="$HOST" + # EXTRA_CONFIGS=( + # "azureServiceUrl=$AZURE_SERVICE_URL" + # ) + # initializeLlmProviderConfig azure azure AZURE "$AZURE_SERVICE_DOMAIN" "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}" if [ -z "$CLAUDE_VERSION" ]; then CLAUDE_VERSION="2023-06-01" @@ -49,22 +49,22 @@ function initializeLlmProviderConfigs() { EXTRA_CONFIGS=( "claudeVersion=\"$CLAUDE_VERSION\"" ) - initializeLlmProviderConfig claude claude CLAUDE api.anthropic.com "443" "https" "" "${EXTRA_CONFIGS[@]}" - - if [ -z "$OLLAMA_SERVER_HOST" ]; then - OLLAMA_SERVER_HOST="YOUR_OLLAMA_SERVER_HOST" - fi - OLLAMA_SERVER_PORT="${OLLAMA_SERVER_PORT:-11434}" - EXTRA_CONFIGS=( - "ollamaServerHost=\"$OLLAMA_SERVER_HOST\"" - "ollamaServerPort=$OLLAMA_SERVER_PORT" - ) - initializeLlmProviderConfig ollama ollama OLLAMA "$OLLAMA_SERVER_HOST" "$OLLAMA_SERVER_PORT" "http" "" "${EXTRA_CONFIGS[@]}" + initializeLlmProviderConfig claude claude CLAUDE api.anthropic.com "443" "https" "" "PRE" "claude-" "${EXTRA_CONFIGS[@]}" + + # if [ -z "$OLLAMA_SERVER_HOST" ]; then + # OLLAMA_SERVER_HOST="YOUR_OLLAMA_SERVER_HOST" + # fi + # OLLAMA_SERVER_PORT="${OLLAMA_SERVER_PORT:-11434}" + # EXTRA_CONFIGS=( + # "ollamaServerHost=\"$OLLAMA_SERVER_HOST\"" + # "ollamaServerPort=$OLLAMA_SERVER_PORT" + # ) + # initializeLlmProviderConfig ollama ollama OLLAMA "$OLLAMA_SERVER_HOST" "$OLLAMA_SERVER_PORT" "http" "" "PRE" "" "${EXTRA_CONFIGS[@]}" EXTRA_CONFIGS=( "minimaxGroupId=\"$MINIMAX_GROUP_ID\"" ) - initializeLlmProviderConfig minimax minimax MINIMAX api.minimax.chat "443" "https" "" "${EXTRA_CONFIGS[@]}" + initializeLlmProviderConfig minimax minimax MINIMAX api.minimax.chat "443" "https" "" "PRE" "abab" "${EXTRA_CONFIGS[@]}" } function initializeLlmProviderConfig() { @@ -82,12 +82,16 @@ function initializeLlmProviderConfig() { shift local DEFAULT_API_KEY="$1" shift + local MODEL_MATCH_TYPE="$1" + shift + local MODEL_MATCH_VALUE="$1" + shift local EXTRA_CONFIGS=("$@") appendAiRegistry "$NAME" "$DOMAIN" "$PORT" "$PROTOCOL" appendAiProxyConfigs "$NAME" "$TYPE" "$API_KEY_PREFIX" "$DEFAULT_API_KEY" "${EXTRA_CONFIGS[@]}" - generateAiIngress "$NAME" - generateAiRoute "$NAME" + generateAiIngress "$NAME" "$MODEL_MATCH_TYPE" "$MODEL_MATCH_VALUE" + generateAiRoute "$NAME" "$MODEL_MATCH_TYPE" "$MODEL_MATCH_VALUE" } function initializeSharedConfigs() { @@ -142,12 +146,11 @@ metadata: resourceVersion: \"1\" spec: defaultConfig: - enable: true - add_header_key: x-higress-llm-provider + modelToHeader: x-higress-llm-model defaultConfigDisable: false failStrategy: FAIL_OPEN - phase: UNSPECIFIED_PHASE - priority: 260 + phase: AUTHN + priority: 900 url: oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/model-router:$MODEL_ROUTER_VERSION" >"$WASM_PLUGIN_CONFIG_FOLDER/model-router.internal.yaml" } @@ -244,20 +247,29 @@ function appendAiRegistry() { function generateAiIngress() { local PROVIDER_NAME="$1" + local MODEL_MATCH_TYPE="$2" + local MODEL_MATCH_VALUE="$3" local INGRESS_NAME="ai-route-$PROVIDER_NAME.internal" local INGRESS_FILE="/data/ingresses/$INGRESS_NAME.yaml" mkdir -p /data/ingresses + local HEADER_MATCH_ANNOTATION_PREFIX="unknown" + if [ "$MODEL_MATCH_TYPE" == "PRE" ]; then + HEADER_MATCH_ANNOTATION_PREFIX="prefix" + elif [ "$MODEL_MATCH_TYPE" == "REGULAR" ]; then + HEADER_MATCH_ANNOTATION_PREFIX="regex" + fi + cat <"$INGRESS_FILE" apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: - higress.io/destination: $LAST_SERVICE_NAME:$LAST_SERVICE_PORT + higress.io/destination: "$LAST_SERVICE_NAME:$LAST_SERVICE_PORT" higress.io/ignore-path-case: "false" - higress.io/exact-match-header-x-higress-llm-provider: $PROVIDER_NAME + higress.io/$HEADER_MATCH_ANNOTATION_PREFIX-match-header-x-higress-llm-model: "$MODEL_MATCH_VALUE" labels: higress.io/resource-definer: higress name: $INGRESS_NAME @@ -280,8 +292,10 @@ EOF function generateAiRoute() { local ROUTE_NAME="$1" + local MODEL_MATCH_TYPE="$2" + local MODEL_MATCH_VALUE="$3" - local CONFIG_MAP_NAME="ai-route-$ROUTE_NAME.internal" + local CONFIG_MAP_NAME="ai-route-$ROUTE_NAME" local CONFIG_MAP_FILE="/data/configmaps/$CONFIG_MAP_NAME.yaml" mkdir -p /data/configmaps @@ -305,10 +319,12 @@ data: "provider": "$ROUTE_NAME" } ], - "modelPredicate": { - "enabled": true, - "prefix": "$ROUTE_NAME" - }, + "modelPredicates": [ + { + "matchType": "$MODEL_MATCH_TYPE", + "matchValue": "$MODEL_MATCH_VALUE" + } + ], "version": "1" } EOF diff --git a/compose/.env b/compose/.env index c202e50..8f1d089 100644 --- a/compose/.env +++ b/compose/.env @@ -8,11 +8,11 @@ NACOS_PASSWORD='' NACOS_DATA_ENC_KEY='' NACOS_SERVER_TAG='v2.2.3' HIGRESS_RUNNER_TAG='0.0.3' -HIGRESS_API_SERVER_TAG='0.0.18' -HIGRESS_CONTROLLER_TAG='2.0.4' -HIGRESS_PILOT_TAG='2.0.4' -HIGRESS_GATEWAY_TAG='2.0.4' -HIGRESS_CONSOLE_TAG='1.4.6' +HIGRESS_API_SERVER_TAG='0.0.19' +HIGRESS_CONTROLLER_TAG='2.0.6-rc.1' +HIGRESS_PILOT_TAG='743ae53ed47f50e9b1f7acc3fdd8ced8068cf3ee' +HIGRESS_GATEWAY_TAG='2.0.6-rc.1' +HIGRESS_CONSOLE_TAG='2.0.0' PROMETHEUS_TAG='v2.40.7' PROMTAIL_TAG='2.9.4' LOKI_TAG='2.9.4' diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 259ea83..a625e93 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -129,7 +129,7 @@ services: networks: higress-net: aliases: - - higress.svc + - controller.svc restart: always healthcheck: test: [ "CMD", "curl", "-f", "http://127.0.0.1:8888/ready" ]