Fix several known bugs related to AI Gateway (higress-group#121)

* rel: Upgrade to Higress 2.0.5

* feat: Update ai-gateway configuration scripts

* Update higress core versions to 2.0.6-rc.1

* Upgrade apiserver to 0.0.19

* Update pilot tag to 743ae53ed47f50e9b1f7acc3fdd8ced8068cf3ee

all-in-one/Dockerfile

@@ -1,16 +1,17 @@
 ARG HUB=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress
 ARG BASE_VERSION=2022-10-27T19-02-22
-ARG CORE_VERSION=2.0.4
-ARG CONSOLE_VERSION=1.4.6
-ARG APISERVER_VERSION=0.0.16
+ARG CORE_VERSION=2.0.6-rc.1
+ARG CONSOLE_VERSION=2.0.0
+ARG APISERVER_VERSION=0.0.19
 ARG PROMETHEUS_VERSION=v2.40.7
 ARG PROMTAIL_VERSION=2.9.4
 ARG LOKI_VERSION=2.9.4
 ARG GRAFANA_VERSION=9.3.6
 
 FROM ${HUB}/api-server:${APISERVER_VERSION} AS apiserver
 FROM ${HUB}/higress:${CORE_VERSION} AS controller
-FROM ${HUB}/pilot:${CORE_VERSION} AS pilot
+# FROM ${HUB}/pilot:${CORE_VERSION} AS pilot
+FROM ${HUB}/pilot:743ae53ed47f50e9b1f7acc3fdd8ced8068cf3ee AS pilot
 FROM ${HUB}/gateway:${CORE_VERSION} AS gateway
 FROM ${HUB}/console:${CONSOLE_VERSION} AS console
 FROM ${HUB}/prometheus:${PROMETHEUS_VERSION} AS prometheus

all-in-one/get-ai-gateway.sh

@@ -146,28 +146,28 @@ runConfigWizard() {
     "OpenAI|OPENAI"
     "Aliyun Dashscope|DASHSCOPE"
     "Moonshot|MOONSHOT"
-    "Azure OpenAI|AZURE|configureAzureProvider"
-    "360 Zhinao|AI360"
-    "Github Models|GITHUB"
-    "Groq|GROQ"
+    # "Azure OpenAI|AZURE|configureAzureProvider"
+    # "360 Zhinao|AI360"
+    # "Github Models|GITHUB"
+    # "Groq|GROQ"
     "Baichuan AI|BAICHUAN"
     "01.AI|YI"
-    "DeepSeek|DEEPSEEK"
+    # "DeepSeek|DEEPSEEK"
     "Zhipu AI|ZHIPUAI"
-    "Ollama|OLLAMA|configureOllamaProvider"
+    # "Ollama|OLLAMA|configureOllamaProvider"
     "Claude|CLAUDE|configureClaudeProvider"
     # "Baidu AI Cloud|BAIDU"
     # "Tencent Hunyuan|HUNYUAN"
     "Stepfun|STEPFUN"
     "Minimax|MINIMAX|configureMinimaxProvider"
     # "Cloudflare Workers AI|CLOUDFLARE"
     # "iFlyTek Spark|SPARK"
-    "Google Gemini|GEMINI"
+    # "Google Gemini|GEMINI"
     # "DeepL|DEEPL"
-    "Mistral AI|MISTRAL"
-    "Cohere|COHERE"
+    # "Mistral AI|MISTRAL"
+    # "Cohere|COHERE"
     "Doubao|DOUBAO"
-    "Coze|COZE"
+    # "Coze|COZE"
   )
 
   local selectedIndex=''

all-in-one/scripts/config-template/ai-gateway.sh

@@ -5,66 +5,66 @@ ROOT=$(pwd)
 cd - >/dev/null
 source $ROOT/../base.sh
 
-AI_PROXY_VERSION=${AI_PROXY_VERSION:-latest}
-MODEL_ROUTER_VERSION=${MODEL_ROUTER_VERSION:-latest}
+AI_PROXY_VERSION=${AI_PROXY_VERSION:-1.0.0}
+MODEL_ROUTER_VERSION=${MODEL_ROUTER_VERSION:-1.0.0}
 
 function initializeLlmProviderConfigs() {
   local EXTRA_CONFIGS=()
 
-  initializeLlmProviderConfig aliyun qwen DASHSCOPE dashscope.aliyuncs.com
-  initializeLlmProviderConfig moonshot moonshot MOONSHOT api.moonshot.cn
-  initializeLlmProviderConfig openai openai OPENAI api.openai.com
-  initializeLlmProviderConfig ai360 ai360 AI360 api.360.cn
-  initializeLlmProviderConfig github github GITHUB models.inference.ai.azure.com
-  initializeLlmProviderConfig groq groq GROQ api.groq.com
-  initializeLlmProviderConfig baichuan baichuan BAICHUAN api.baichuan-ai.com
-  initializeLlmProviderConfig yi yi YI api.lingyiwanwu.com
-  initializeLlmProviderConfig deepseek deepseek DEEPSEEK api.deepseek.com
-  initializeLlmProviderConfig zhipuai zhipuai ZHIPUAI open.bigmodel.cn
-  # initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com
-  # initializeLlmProviderConfig hunyuan hunyuan HUNYUAN hunyuan.tencentcloudapi.com 443 "https" "" "${EXTRA_CONFIGS[@]}"
-  initializeLlmProviderConfig stepfun stepfun STEPFUN api.stepfun.com
-  # initializeLlmProviderConfig cloudflare cloudflare CLOUDFLARE api.cloudflare.com 443 "https" "" "${EXTRA_CONFIGS[@]}"
-  # initializeLlmProviderConfig spark spark SPARK spark-api-open.xf-yun.com 443 "https" "" "${EXTRA_CONFIGS[@]}"
-  initializeLlmProviderConfig gemini gemini GEMINI generativelanguage.googleapis.com
-  # initializeLlmProviderConfig deepl deepl DEEPL 443 "https" "" "${EXTRA_CONFIGS[@]}"
-  initializeLlmProviderConfig mistral mistral MISTRAL api.mistral.ai
-  initializeLlmProviderConfig cohere cohere COHERE api.cohere.com
-  initializeLlmProviderConfig doubao doubao DOUBAO ark.cn-beijing.volces.com
-  initializeLlmProviderConfig coze coze COZE api.coze.cn
-
-  if [ -z "$AZURE_SERVICE_URL" ]; then
-    AZURE_SERVICE_URL="https://YOUR_RESOURCE_NAME.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT_NAME/chat/completions?api-version=2024-06-01"
-  fi
-  extractHostFromUrl "$AZURE_SERVICE_URL"
-  local AZURE_SERVICE_DOMAIN="$HOST"
-  EXTRA_CONFIGS=(
-    "azureServiceUrl=$AZURE_SERVICE_URL"
-  )
-  initializeLlmProviderConfig azure azure AZURE "$AZURE_SERVICE_DOMAIN" "443" "https" "" "${EXTRA_CONFIGS[@]}"
+  initializeLlmProviderConfig aliyun qwen DASHSCOPE dashscope.aliyuncs.com "443" "https" "" "PRE" 'qwen-'
+  initializeLlmProviderConfig moonshot moonshot MOONSHOT api.moonshot.cn "443" "https" "" "PRE" 'moonshot-'
+  initializeLlmProviderConfig openai openai OPENAI api.openai.com "443" "https" "" "REGULAR" 'gpt\\-.*|o1\\-.*'
+  # initializeLlmProviderConfig ai360 ai360 AI360 api.360.cn "443" "https" "" "PRE" ""
+  # initializeLlmProviderConfig github github GITHUB models.inference.ai.azure.com "443" "https" "" "PRE" ""
+  # initializeLlmProviderConfig groq groq GROQ api.groq.com "443" "https" "" "PRE" ""
+  initializeLlmProviderConfig baichuan baichuan BAICHUAN api.baichuan-ai.com "443" "https" "" "PRE" "Baichuan"
+  initializeLlmProviderConfig yi yi YI api.lingyiwanwu.com "443" "https" "" "PRE" 'yi-'
+  initializeLlmProviderConfig deepseek deepseek DEEPSEEK api.deepseek.com "443" "https" "" "PRE" "deepseek"
+  initializeLlmProviderConfig zhipuai zhipuai ZHIPUAI open.bigmodel.cn "443" "https" "" "PRE" "GLM-"
+  # initializeLlmProviderConfig baidu baidu BAIDU aip.baidubce.com "443" "https" "" "PRE" "ERNIE-"
+  # initializeLlmProviderConfig hunyuan hunyuan HUNYUAN hunyuan.tencentcloudapi.com "443" "https" "" "PRE" "hunyuan-" "${EXTRA_CONFIGS[@]}"
+  initializeLlmProviderConfig stepfun stepfun STEPFUN api.stepfun.com "443" "https" "" "PRE" "step-"
+  # initializeLlmProviderConfig cloudflare cloudflare CLOUDFLARE api.cloudflare.com "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}"
+  # initializeLlmProviderConfig spark spark SPARK spark-api-open.xf-yun.com "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}"
+  # initializeLlmProviderConfig gemini gemini GEMINI generativelanguage.googleapis.com "443" "https" "" "PRE" "gemini-"
+  # initializeLlmProviderConfig deepl deepl DEEPL "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}"
+  # initializeLlmProviderConfig mistral mistral MISTRAL api.mistral.ai "443" "https" "" "PRE" ""
+  # initializeLlmProviderConfig cohere cohere COHERE api.cohere.com "443" "https" "" "PRE" ""
+  initializeLlmProviderConfig doubao doubao DOUBAO ark.cn-beijing.volces.com "443" "https" "" "PRE" "doubao-"
+  # initializeLlmProviderConfig coze coze COZE api.coze.cn "443" "https" "" "PRE" ""
+
+  # if [ -z "$AZURE_SERVICE_URL" ]; then
+  #   AZURE_SERVICE_URL="https://YOUR_RESOURCE_NAME.openai.azure.com/openai/deployments/YOUR_DEPLOYMENT_NAME/chat/completions?api-version=2024-06-01"
+  # fi
+  # extractHostFromUrl "$AZURE_SERVICE_URL"
+  # local AZURE_SERVICE_DOMAIN="$HOST"
+  # EXTRA_CONFIGS=(
+  #   "azureServiceUrl=$AZURE_SERVICE_URL"
+  # )
+  # initializeLlmProviderConfig azure azure AZURE "$AZURE_SERVICE_DOMAIN" "443" "https" "" "PRE" "" "${EXTRA_CONFIGS[@]}"
 
   if [ -z "$CLAUDE_VERSION" ]; then
     CLAUDE_VERSION="2023-06-01"
   fi
   EXTRA_CONFIGS=(
     "claudeVersion=\"$CLAUDE_VERSION\""
   )
-  initializeLlmProviderConfig claude claude CLAUDE api.anthropic.com "443" "https" "" "${EXTRA_CONFIGS[@]}"
-
-  if [ -z "$OLLAMA_SERVER_HOST" ]; then
-    OLLAMA_SERVER_HOST="YOUR_OLLAMA_SERVER_HOST"
-  fi
-  OLLAMA_SERVER_PORT="${OLLAMA_SERVER_PORT:-11434}"
-  EXTRA_CONFIGS=(
-    "ollamaServerHost=\"$OLLAMA_SERVER_HOST\""
-    "ollamaServerPort=$OLLAMA_SERVER_PORT"
-  )
-  initializeLlmProviderConfig ollama ollama OLLAMA "$OLLAMA_SERVER_HOST" "$OLLAMA_SERVER_PORT" "http" "" "${EXTRA_CONFIGS[@]}"
+  initializeLlmProviderConfig claude claude CLAUDE api.anthropic.com "443" "https" "" "PRE" "claude-" "${EXTRA_CONFIGS[@]}"
+
+  # if [ -z "$OLLAMA_SERVER_HOST" ]; then
+  #   OLLAMA_SERVER_HOST="YOUR_OLLAMA_SERVER_HOST"
+  # fi
+  # OLLAMA_SERVER_PORT="${OLLAMA_SERVER_PORT:-11434}"
+  # EXTRA_CONFIGS=(
+  #   "ollamaServerHost=\"$OLLAMA_SERVER_HOST\""
+  #   "ollamaServerPort=$OLLAMA_SERVER_PORT"
+  # )
+  # initializeLlmProviderConfig ollama ollama OLLAMA "$OLLAMA_SERVER_HOST" "$OLLAMA_SERVER_PORT" "http" "" "PRE" "" "${EXTRA_CONFIGS[@]}"
 
   EXTRA_CONFIGS=(
     "minimaxGroupId=\"$MINIMAX_GROUP_ID\""
   )
-  initializeLlmProviderConfig minimax minimax MINIMAX api.minimax.chat "443" "https" "" "${EXTRA_CONFIGS[@]}"
+  initializeLlmProviderConfig minimax minimax MINIMAX api.minimax.chat "443" "https" "" "PRE" "abab" "${EXTRA_CONFIGS[@]}"
 }
 
 function initializeLlmProviderConfig() {
@@ -82,12 +82,16 @@ function initializeLlmProviderConfig() {
   shift
   local DEFAULT_API_KEY="$1"
   shift
+  local MODEL_MATCH_TYPE="$1"
+  shift
+  local MODEL_MATCH_VALUE="$1"
+  shift
   local EXTRA_CONFIGS=("$@")
 
   appendAiRegistry "$NAME" "$DOMAIN" "$PORT" "$PROTOCOL"
   appendAiProxyConfigs "$NAME" "$TYPE" "$API_KEY_PREFIX" "$DEFAULT_API_KEY" "${EXTRA_CONFIGS[@]}"
-  generateAiIngress "$NAME"
-  generateAiRoute "$NAME"
+  generateAiIngress "$NAME" "$MODEL_MATCH_TYPE" "$MODEL_MATCH_VALUE"
+  generateAiRoute "$NAME" "$MODEL_MATCH_TYPE" "$MODEL_MATCH_VALUE"
 }
 
 function initializeSharedConfigs() {
@@ -142,12 +146,11 @@ metadata:
   resourceVersion: \"1\"
 spec:
   defaultConfig:
-    enable: true
-    add_header_key: x-higress-llm-provider
+    modelToHeader: x-higress-llm-model
   defaultConfigDisable: false
   failStrategy: FAIL_OPEN
-  phase: UNSPECIFIED_PHASE
-  priority: 260
+  phase: AUTHN
+  priority: 900
   url: oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/model-router:$MODEL_ROUTER_VERSION" >"$WASM_PLUGIN_CONFIG_FOLDER/model-router.internal.yaml"
 }
 
@@ -244,20 +247,29 @@ function appendAiRegistry() {
 
 function generateAiIngress() {
   local PROVIDER_NAME="$1"
+  local MODEL_MATCH_TYPE="$2"
+  local MODEL_MATCH_VALUE="$3"
 
   local INGRESS_NAME="ai-route-$PROVIDER_NAME.internal"
   local INGRESS_FILE="/data/ingresses/$INGRESS_NAME.yaml"
 
   mkdir -p /data/ingresses
 
+  local HEADER_MATCH_ANNOTATION_PREFIX="unknown"
+  if [ "$MODEL_MATCH_TYPE" == "PRE" ]; then
+    HEADER_MATCH_ANNOTATION_PREFIX="prefix"
+  elif [ "$MODEL_MATCH_TYPE" == "REGULAR" ]; then
+    HEADER_MATCH_ANNOTATION_PREFIX="regex"
+  fi
+
   cat <<EOF >"$INGRESS_FILE"
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   annotations:
-    higress.io/destination: $LAST_SERVICE_NAME:$LAST_SERVICE_PORT
+    higress.io/destination: "$LAST_SERVICE_NAME:$LAST_SERVICE_PORT"
     higress.io/ignore-path-case: "false"
-    higress.io/exact-match-header-x-higress-llm-provider: $PROVIDER_NAME
+    higress.io/$HEADER_MATCH_ANNOTATION_PREFIX-match-header-x-higress-llm-model: "$MODEL_MATCH_VALUE"
   labels:
     higress.io/resource-definer: higress
   name: $INGRESS_NAME
@@ -280,8 +292,10 @@ EOF
 
 function generateAiRoute() {
   local ROUTE_NAME="$1"
+  local MODEL_MATCH_TYPE="$2"
+  local MODEL_MATCH_VALUE="$3"
 
-  local CONFIG_MAP_NAME="ai-route-$ROUTE_NAME.internal"
+  local CONFIG_MAP_NAME="ai-route-$ROUTE_NAME"
   local CONFIG_MAP_FILE="/data/configmaps/$CONFIG_MAP_NAME.yaml"
 
   mkdir -p /data/configmaps
@@ -305,10 +319,12 @@ data:
           "provider": "$ROUTE_NAME"
         }
       ],
-      "modelPredicate": {
-        "enabled": true,
-        "prefix": "$ROUTE_NAME"
-      },
+      "modelPredicates": [
+        {
+          "matchType": "$MODEL_MATCH_TYPE",
+          "matchValue": "$MODEL_MATCH_VALUE"
+        }
+      ],
       "version": "1"
     }
 EOF

compose/.env

@@ -8,11 +8,11 @@ NACOS_PASSWORD=''
 NACOS_DATA_ENC_KEY=''
 NACOS_SERVER_TAG='v2.2.3'
 HIGRESS_RUNNER_TAG='0.0.3'
-HIGRESS_API_SERVER_TAG='0.0.18'
-HIGRESS_CONTROLLER_TAG='2.0.4'
-HIGRESS_PILOT_TAG='2.0.4'
-HIGRESS_GATEWAY_TAG='2.0.4'
-HIGRESS_CONSOLE_TAG='1.4.6'
+HIGRESS_API_SERVER_TAG='0.0.19'
+HIGRESS_CONTROLLER_TAG='2.0.6-rc.1'
+HIGRESS_PILOT_TAG='743ae53ed47f50e9b1f7acc3fdd8ced8068cf3ee'
+HIGRESS_GATEWAY_TAG='2.0.6-rc.1'
+HIGRESS_CONSOLE_TAG='2.0.0'
 PROMETHEUS_TAG='v2.40.7'
 PROMTAIL_TAG='2.9.4'
 LOKI_TAG='2.9.4'

compose/docker-compose.yml

@@ -129,7 +129,7 @@ services:
     networks:
       higress-net:
         aliases:
-          - higress.svc
+          - controller.svc
     restart: always
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://127.0.0.1:8888/ready" ]