From a53b4707fb03efa40966a00f1525bdedad2334ce Mon Sep 17 00:00:00 2001 From: nic-chen <33000667+nic-chen@users.noreply.github.com> Date: Mon, 27 Apr 2020 10:47:17 +0800 Subject: [PATCH] feat: support saving k8s deployment info to upstream (#1502) * feat: support save k8s deployment info to upstream --- apisix/admin/upstreams.lua | 12 +++--- apisix/schema_def.lua | 28 +++++++++++-- doc/admin-api-cn.md | 10 ++++- doc/admin-api.md | 10 ++++- doc/architecture-design-cn.md | 13 +++--- doc/architecture-design.md | 13 +++--- t/admin/routes.t | 4 +- t/admin/services.t | 6 +-- t/admin/upstream.t | 4 +- t/node/upstream.t | 76 +++++++++++++++++++++++++++++++---- 10 files changed, 142 insertions(+), 34 deletions(-) diff --git a/apisix/admin/upstreams.lua b/apisix/admin/upstreams.lua index b49c33a78ec5..e989cd552783 100644 --- a/apisix/admin/upstreams.lua +++ b/apisix/admin/upstreams.lua @@ -19,7 +19,6 @@ local get_routes = require("apisix.router").http_routes local get_services = require("apisix.http.service").services local tostring = tostring local ipairs = ipairs -local tonumber = tonumber local type = type @@ -99,17 +98,20 @@ local function check_conf(id, conf, need_id) if need_id and conf.id and tostring(conf.id) ~= tostring(id) then return nil, {error_msg = "wrong upstream id"} end + + -- let schema check id + if id and not conf.id then + conf.id = id + end + core.log.info("schema: ", core.json.delay_encode(core.schema.upstream)) core.log.info("conf : ", core.json.delay_encode(conf)) + local ok, err = check_upstream_conf(conf) if not ok then return nil, {error_msg = err} end - if need_id and not tonumber(id) then - return nil, {error_msg = "wrong type of service id"} - end - return need_id and id or true end diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua index 50a783acd599..e261c98c1ec4 100644 --- a/apisix/schema_def.lua +++ b/apisix/schema_def.lua @@ -29,8 +29,8 @@ local plugins_schema = { local id_schema = { anyOf = { { - type = "string", minLength = 1, maxLength = 32, - pattern = [[^[0-9]+$]] + type = "string", minLength = 1, maxLength = 64, + pattern = [[^[a-zA-Z0-9-_]+$]] }, {type = "integer", minimum = 1} } @@ -253,6 +253,25 @@ local upstream_schema = { }, required = {"connect", "send", "read"}, }, + k8s_deployment_info = { + type = "object", + properties = { + namespace = {type = "string", description = "k8s namespace"}, + deploy_name = {type = "string", description = "k8s deployment name"}, + service_name = {type = "string", description = "k8s service name"}, + port = {type = "number", minimum = 0}, + backend_type = { + type = "string", + default = "pod", + description = "k8s service name", + enum = {"svc", "pod"} + }, + }, + anyOf = { + {required = {"namespace", "deploy_name", "port"}}, + {required = {"namespace", "service_name", "port"}}, + }, + }, type = { description = "algorithms of load balancing", type = "string", @@ -280,7 +299,10 @@ local upstream_schema = { desc = {type = "string", maxLength = 256}, id = id_schema }, - required = {"nodes", "type"}, + anyOf = { + {required = {"type", "nodes"}}, + {required = {"type", "k8s_deployment_info"}}, + }, additionalProperties = false, } diff --git a/doc/admin-api-cn.md b/doc/admin-api-cn.md index d61bbe9f47e9..7a7d3555b27c 100644 --- a/doc/admin-api-cn.md +++ b/doc/admin-api-cn.md @@ -344,7 +344,8 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外，还支持对上 |名字 |可选项 |类型 |说明 |示例| |---------|---------|----|-----------|----| -|nodes |必需|Node|哈希表，内部元素的 key 是上游机器地址列表，格式为`地址 + Port`，其中地址部分可以是 IP 也可以是域名，比如 `192.168.1.100:80`、`foo.com:80`等。value 则是节点的权重，特别的，当权重值为 `0` 有特殊含义，通常代表该上游节点失效，永远不希望被选中。|`192.168.1.100:80`| +|nodes |与 `k8s_deployment_info` 二选一|Node|哈希表，内部元素的 key 是上游机器地址列表，格式为`地址 + Port`，其中地址部分可以是 IP 也可以是域名，比如 `192.168.1.100:80`、`foo.com:80`等。value 则是节点的权重，特别的，当权重值为 `0` 有特殊含义，通常代表该上游节点失效，永远不希望被选中。|`192.168.1.100:80`| +|k8s_deployment_info|与 `nodes` 二选一|哈希表|字段包括 `namespace`、`deploy_name`、`service_name`、`port`、`backend_type`，其中 `port` 字段为数值，`backend_type` 为 `pod` 或 `service`，其他为字符串 | `{"namespace": "test-namespace", "deploy_name": "test-deploy-name", "service_name": "test-service-name", "backend_type": "pod", "port": 8080}` | |type |必需|枚举|`roundrobin` 支持权重的负载，`chash` 一致性哈希，两者是二选一的|`roundrobin`|| |key |条件必需|匹配类型|该选项只有类型是 `chash` 才有效。根据 `key` 来查找对应的 node `id`，相同的 `key` 在同一个对象中，永远返回相同 id，目前支持的 Nginx 内置变量有 `uri, server_name, server_addr, request_uri, remote_port, remote_addr, query_string, host, hostname, arg_***`，其中 `arg_***` 是来自URL的请求参数，[Nginx 变量列表](http://nginx.org/en/docs/varindex.html)|| |checks |可选|health_checker|配置健康检查的参数，详细可参考[health-check](health-check.md)|| @@ -367,6 +368,13 @@ upstream 对象 json 配置内容： }, "enable_websocket": true, "nodes": {"host:80": 100}, # 上游机器地址列表，格式为`地址 + Port` + "k8s_deployment_info": { # k8s deployment 信息 + "namespace": "test-namespace", + "deploy_name": "test-deploy-name", + "service_name": "test-service-name", + "backend_type": "pod", # pod or service + "port": 8080 + }, "type":"roundrobin", # chash or roundrobin "checks": {}, # 配置健康检查的参数 "hash_on": "", diff --git a/doc/admin-api.md b/doc/admin-api.md index 9c40948b35d8..f60ccb6aef0e 100644 --- a/doc/admin-api.md +++ b/doc/admin-api.md @@ -337,7 +337,8 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst |Name |Optional|Description| |------- |-----|------| |type |required|`roundrobin` supports the weight of the load, `chash` consistency hash, pick one of them.| -|nodes |required|Hash table, the key of the internal element is the upstream machine address list, the format is `Address + Port`, where the address part can be IP or domain name, such as `192.168.1.100:80`, `foo.com:80`, etc. Value is the weight of the node. In particular, when the weight value is `0`, it has a special meaning, which usually means that the upstream node is invalid and never wants to be selected.| +|nodes |required if `k8s_deployment_info` not configured|Hash table, the key of the internal element is the upstream machine address list, the format is `Address + Port`, where the address part can be IP or domain name, such as `192.168.1.100:80`, `foo.com:80`, etc. Value is the weight of the node. In particular, when the weight value is `0`, it has a special meaning, which usually means that the upstream node is invalid and never wants to be selected.| +|k8s_deployment_info|required if `nodes` not configured|fields: `namespace`、`deploy_name`、`service_name`、`port`、`backend_type`, `port` is number, `backend_type` is `pod` or `service`, others is string. | |hash_on |optional|This option is only valid if the `type` is `chash`. Supported types `vars`(Nginx variables), `header`(custom header), `cookie`, `consumer`, the default value is `vars`.| |key |required|This option is only valid if the `type` is `chash`. Find the corresponding node `id` according to `hash_on` and `key`. When `hash_on` is set as `vars`, `key` is the required parameter, for now, it support nginx built-in variables like `uri, server_name, server_addr, request_uri, remote_port, remote_addr, query_string, host, hostname, arg_***`, `arg_***` is arguments in the request line, [Nginx variables list](http://nginx.org/en/docs/varindex.html). When `hash_on` is set as `header`, `key` is the required parameter, and `header name` is customized. When `hash_on` is set to `cookie`, `key` is the required parameter, and `cookie name` is customized. When `hash_on` is set to `consumer`, `key` does not need to be set. In this case, the `key` adopted by the hash algorithm is the `consumer_id` authenticated. If the specified `hash_on` and `key` can not fetch values, it will be fetch `remote_addr` by default.| |checks |optional|Configure the parameters of the health check. For details, refer to [health-check](health-check.md).| @@ -359,6 +360,13 @@ Config Example: }, "enable_websocket": true, "nodes": {"host:80": 100}, # Upstream machine address list, the format is `Address + Port` + "k8s_deployment_info": { # kubernetes deployment info + "namespace": "test-namespace", + "deploy_name": "test-deploy-name", + "service_name": "test-service-name", + "backend_type": "pod", # pod or service + "port": 8080 + }, "type":"roundrobin", # chash or roundrobin "checks": {}, # Health check parameters "hash_on": "", diff --git a/doc/architecture-design-cn.md b/doc/architecture-design-cn.md index 251a0f37df58..3fa688448fe3 100644 --- a/doc/architecture-design-cn.md +++ b/doc/architecture-design-cn.md @@ -238,7 +238,8 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外，还支持对上 |名字 |可选|说明| |------- |-----|------| |type |必填|`roundrobin` 支持权重的负载，`chash` 一致性哈希，两者是二选一的| -|nodes |必填|哈希表，内部元素的 key 是上游机器地址列表，格式为`地址 + Port`，其中地址部分可以是 IP 也可以是域名，比如 `192.168.1.100:80`、`foo.com:80` 等。value 则是节点的权重。当权重值为 `0` 代表该上游节点失效，不会被选中，可以用于暂时摘除节点的情况。| +|nodes |与 `k8s_deployment_info` 二选一|哈希表，内部元素的 key 是上游机器地址列表，格式为`地址 + Port`，其中地址部分可以是 IP 也可以是域名，比如 `192.168.1.100:80`、`foo.com:80` 等。value 则是节点的权重。当权重值为 `0` 代表该上游节点失效，不会被选中，可以用于暂时摘除节点的情况。| +|k8s_deployment_info|与 `nodes` 二选一|哈希表|字段包括 `namespace`、`deploy_name`、`service_name`、`port`、`backend_type`，其中 `port` 字段为数值，`backend_type` 为 `pod` 或 `service`，其他为字符串 | |key |可选|在 `type` 等于 `chash` 是必选项。 `key` 需要配合 `hash_on` 来使用，通过 `hash_on` 和 `key` 来查找对应的 node `id`| |hash_on |可选|`hash_on` 支持的类型有 `vars`（Nginx内置变量），`header`（自定义header），`cookie`，`consumer`，默认值为 `vars`| |checks |可选|配置健康检查的参数，详细可参考[health-check](health-check.md)| @@ -259,10 +260,12 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外，还支持对上 curl http://127.0.0.1:9080/apisix/admin/upstreams/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' { "type": "roundrobin", - "nodes": { - "127.0.0.1:80": 1, - "127.0.0.2:80": 2, - "foo.com:80": 3 + "k8s_deployment_info": { + "namespace": "test-namespace", + "deploy_name": "test-deploy-name", + "service_name": "test-service-name", + "backend_type": "pod", + "port": 8080 } }' diff --git a/doc/architecture-design.md b/doc/architecture-design.md index e580ab37a5fd..9edaadb378b4 100644 --- a/doc/architecture-design.md +++ b/doc/architecture-design.md @@ -233,7 +233,8 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst |Name |Optional|Description| |------- |-----|------| |type |required|`roundrobin` supports the weight of the load, `chash` consistency hash, pick one of them.| -|nodes |required|Hash table, the key of the internal element is the upstream machine address list, the format is `Address + Port`, where the address part can be IP or domain name, such as `192.168.1.100:80`, `foo.com:80`, etc. Value is the weight of the node. In particular, when the weight value is `0`, it has a special meaning, which usually means that the upstream node is invalid and never wants to be selected.| +|nodes |required if `k8s_deployment_info` not configured|Hash table, the key of the internal element is the upstream machine address list, the format is `Address + Port`, where the address part can be IP or domain name, such as `192.168.1.100:80`, `foo.com:80`, etc. Value is the weight of the node. In particular, when the weight value is `0`, it has a special meaning, which usually means that the upstream node is invalid and never wants to be selected.| +|k8s_deployment_info |required if `nodes` not configured|fields: `namespace`、`deploy_name`、`service_name`、`port`、`backend_type`, `port` is number, `backend_type` is `pod` or `service`, others is string. | |hash_on |optional|This option is only valid if the `type` is `chash`. Supported types `vars`(Nginx variables), `header`(custom header), `cookie`, `consumer`, the default value is `vars`.| |key |required|This option is only valid if the `type` is `chash`. Find the corresponding node `id` according to `hash_on` and `key`. When `hash_on` is set as `vars`, `key` is the required parameter, for now, it support nginx built-in variables like `uri, server_name, server_addr, request_uri, remote_port, remote_addr, query_string, host, hostname, arg_***`, `arg_***` is arguments in the request line, [Nginx variables list](http://nginx.org/en/docs/varindex.html). When `hash_on` is set as `header`, `key` is the required parameter, and `header name` is customized. When `hash_on` is set to `cookie`, `key` is the required parameter, and `cookie name` is customized. When `hash_on` is set to `consumer`, `key` does not need to be set. In this case, the `key` adopted by the hash algorithm is the `consumer_id` authenticated. If the specified `hash_on` and `key` can not fetch values, it will be fetch `remote_addr` by default.| |checks |optional|Configure the parameters of the health check. For details, refer to [health-check](health-check.md).| @@ -248,10 +249,12 @@ Create an upstream object use case: curl http://127.0.0.1:9080/apisix/admin/upstreams/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' { "type": "roundrobin", - "nodes": { - "127.0.0.1:80": 1, - "127.0.0.2:80": 2, - "foo.com:80": 3 + "k8s_deployment_info": { + "namespace": "test-namespace", + "deploy_name": "test-deploy-name", + "service_name": "test-service-name", + "backend_type": "pod", + "port": 8080 } }' diff --git a/t/admin/routes.t b/t/admin/routes.t index 4c54c4ffa133..78ab5c62fb6b 100644 --- a/t/admin/routes.t +++ b/t/admin/routes.t @@ -434,7 +434,7 @@ GET /t local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ - "service_id": "invalid_id", + "service_id": "invalid_id$", "uri": "/index.html" }]] ) @@ -569,7 +569,7 @@ GET /t local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, [[{ - "upstream_id": "invalid", + "upstream_id": "invalid$", "uri": "/index.html" }]] ) diff --git a/t/admin/services.t b/t/admin/services.t index 97d0aeac6f19..6c6f5037a69f 100644 --- a/t/admin/services.t +++ b/t/admin/services.t @@ -353,7 +353,7 @@ GET /t location /t { content_by_lua_block { local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/services/invalid_id', + local code, body = t('/apisix/admin/services/invalid_id$', ngx.HTTP_PUT, [[{ "plugins": { @@ -475,7 +475,7 @@ GET /t local code, body = t('/apisix/admin/services', ngx.HTTP_PUT, [[{ - "id": "invalid_id", + "id": "invalid_id$", "plugins": {} }]] ) @@ -530,7 +530,7 @@ GET /t ngx.HTTP_PUT, [[{ "id": 1, - "upstream_id": "invalid" + "upstream_id": "invalid$" }]] ) diff --git a/t/admin/upstream.t b/t/admin/upstream.t index 67359147bb16..da131348a43c 100644 --- a/t/admin/upstream.t +++ b/t/admin/upstream.t @@ -244,7 +244,7 @@ GET /t location /t { content_by_lua_block { local t = require("lib.test_admin").test - local code, body = t('/apisix/admin/upstreams/invalid_id', + local code, body = t('/apisix/admin/upstreams/invalid_id$', ngx.HTTP_PUT, [[{ "nodes": { @@ -374,7 +374,7 @@ GET /t local code, body = t('/apisix/admin/upstreams', ngx.HTTP_PUT, [[{ - "id": "invalid_id", + "id": "invalid_id$", "nodes": { "127.0.0.1:8080": 1 }, diff --git a/t/node/upstream.t b/t/node/upstream.t index bfdbba974734..c6ae077a4ff1 100644 --- a/t/node/upstream.t +++ b/t/node/upstream.t @@ -26,7 +26,69 @@ run_tests(); __DATA__ -=== TEST 1: set upstream(id: 1) +=== TEST 1: set upstream(id: 1) invalid parameters +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + [[{ + "type": "roundrobin", + "desc": "new upstream" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- error_code: 400 +--- no_error_log +[error] + + + +=== TEST 2: set upstream(id: 1) k8s deployment info +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/upstreams/1', + ngx.HTTP_PUT, + [[{ + "k8s_deployment_info": { + "namespace": "test-namespace", + "deploy_name": "test-deploy-name", + "service_name": "test-service-name", + "backend_type": "pod", + "port": 8080 + }, + "type": "roundrobin", + "desc": "new upstream" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- request +GET /t +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 3: set upstream(id: 1) nodes --- config location /t { content_by_lua_block { @@ -57,7 +119,7 @@ passed -=== TEST 2: set route(id: 1) +=== TEST 4: set route(id: 1) --- config location /t { content_by_lua_block { @@ -85,7 +147,7 @@ passed -=== TEST 3: /not_found +=== TEST 5: /not_found --- request GET /not_found --- error_code: 404 @@ -106,7 +168,7 @@ hello world -=== TEST 5: delete upstream(id: 1) +=== TEST 6: delete upstream(id: 1) --- config location /t { content_by_lua_block { @@ -131,7 +193,7 @@ GET /t -=== TEST 6: delete route(id: 1) +=== TEST 7: delete route(id: 1) --- config location /t { content_by_lua_block { @@ -155,7 +217,7 @@ GET /t -=== TEST 7: delete upstream(id: 1) +=== TEST 8: delete upstream(id: 1) --- config location /t { content_by_lua_block { @@ -179,7 +241,7 @@ GET /t -=== TEST 8: delete upstream again(id: 1) +=== TEST 9: delete upstream again(id: 1) --- config location /t { content_by_lua_block {