Tool to fetch API specs. Document API support

Project.toml

@@ -4,7 +4,7 @@ authors = ["JuliaHub Inc."]
 keywords = ["kubernetes", "client"]
 license = "MIT"
 desc = "Julia Kubernetes Client"
-version = "0.7.4"
+version = "0.7.5"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"

README.md

@@ -6,6 +6,8 @@ A Julia Kubernetes Client.
 
 An easy to use API to access Kubernetes clusters from Julia. The `Kuber.ApiImpl.Kubernetes` submodule has the complete set of low level APIs and entities.
 
+[Supported API Versions](SupportedAPIVersions.md)
+
 Most of the low level APIs fit into a common usage pattern. Kuber.jl makes it possible to use all of them with only a few intuitive verb based APIs. Verbs act on entities. Entities can be identified by names or selector patterns, or otherwise can apply to all entities of that class. Verbs can take additional parameters, e.g. when creating or updating entities.
 
 API and Entity naming convention follows the standard Kubernetes API and Model naming conventions.

SupportedAPIVersions.md

@@ -0,0 +1,77 @@
+## Supported API Versions
+
+The default API client included in this version of Kuber.jl supports the following API versions:
+
+- `admissionregistration`
+- `admissionregistration_v1`
+- `admissionregistration_v1beta1`
+- `apiextensions`
+- `apiextensions_v1`
+- `apiextensions_v1beta1`
+- `apiregistration`
+- `apiregistration_v1`
+- `apiregistration_v1beta1`
+- `apis`
+- `apps`
+- `apps_v1`
+- `apps_v1beta1`
+- `apps_v1beta2`
+- `auditregistration`
+- `auditregistration_v1alpha1`
+- `authentication`
+- `authentication_v1`
+- `authentication_v1beta1`
+- `authorization`
+- `authorization_v1`
+- `authorization_v1beta1`
+- `autoscaling`
+- `autoscaling_v1`
+- `autoscaling_v2beta1`
+- `autoscaling_v2beta2`
+- `batch`
+- `batch_v1`
+- `batch_v1beta1`
+- `batch_v2alpha1`
+- `certificates`
+- `certificates_v1beta1`
+- `coordination`
+- `coordination_v1`
+- `coordination_v1beta1`
+- `core`
+- `core_v1`
+- `custom_metrics_v1beta1`
+- `discovery`
+- `discovery_v1beta1`
+- `events`
+- `events_v1beta1`
+- `extensions`
+- `extensions_v1beta1`
+- `flowcontrolApiserver`
+- `flowcontrolApiserver_v1alpha1`
+- `karpenterSh_v1alpha5`
+- `logs`
+- `metrics_v1beta1`
+- `networking`
+- `networking_v1`
+- `networking_v1beta1`
+- `node`
+- `node_v1alpha1`
+- `node_v1beta1`
+- `policy`
+- `policy_v1beta1`
+- `rbacAuthorization`
+- `rbacAuthorization_v1`
+- `rbacAuthorization_v1alpha1`
+- `rbacAuthorization_v1beta1`
+- `scheduling`
+- `scheduling_v1`
+- `scheduling_v1alpha1`
+- `scheduling_v1beta1`
+- `settings`
+- `settings_v1alpha1`
+- `storage`
+- `storage_v1`
+- `storage_v1alpha1`
+- `storage_v1beta1`
+- `version`
+

gen/K8sOpenAPISpec.jl

@@ -0,0 +1,203 @@
+"""
+To consolidate all discovered OpenAPI endpoint specifications:
+- Ensure all required CRDs are applied on the k8s server
+- Preferably use `kubectl proxy` temporarily to avoid having to install certificates
+- Run `julia -e 'include("K8sOpenAPISpec.jl"); K8sOpenAPISpec.get_specification()'`. This should create a file named `k8s.json` with all discovered OpenAPI specifications included.
+
+To display all discovered API versions in a generated `k8s.json` file:
+- Run `julia -e 'include("K8sOpenAPISpec.jl"); K8sOpenAPISpec.show_api_versions()'`
+"""
+module K8sOpenAPISpec
+
+using Downloads
+using JSON
+using YAML
+
+const DEFAULT_API_SERVER = "http://localhost:8001"
+const API_DISCOVERY = "/openapi/v3"
+
+function fetch_json(url::String)
+    iob = IOBuffer()
+    Downloads.download(url, iob)
+    return JSON.parse(String(take!(iob)))
+end
+
+function discover_apis(api_server::String=DEFAULT_API_SERVER)
+    api_discovery_url = string(api_server, API_DISCOVERY)
+    api_discovery = fetch_json(api_discovery_url)
+    api_discovery["paths"]
+end
+
+function merge_l1!(merged_spec::Dict{String,Any}, spec::Dict{String,Any}, key::String)
+    haskey(spec, key) || return
+    if haskey(merged_spec, key)
+        merge!(merged_spec[key], spec[key])
+    else
+        merged_spec[key] = spec[key]
+    end
+    return merged_spec
+end
+
+function merge_l2(merged_spec::Dict{String,Any}, spec::Dict{String,Any}, key1::String)
+    haskey(spec, key1) || return
+
+    if !haskey(merged_spec, key1)
+        merged_spec[key1] = Dict{String,Any}()
+    end
+
+    for key2 in keys(spec[key1])
+        merge_l1!(merged_spec[key1], spec[key1], key2)
+    end
+    return merged_spec
+end
+
+function merge_array!(merged_spec::Dict{String,Any}, spec::Dict{String,Any}, key::String)
+    if haskey(spec, key)
+        if haskey(merged_spec, key)
+            merged_spec[key] = vcat(merged_spec[key], spec[key])
+        else
+            merged_spec[key] = spec[key]
+        end
+    end
+    return merged_spec
+end
+
+function merge_spec!(merged_spec::Dict{String,Any}, spec::Dict{String,Any})
+    # /openapi
+    haskey(spec, "openapi") && (merged_spec["openapi"] = spec["openapi"])
+
+    # /info
+    merge_l1!(merged_spec, spec, "info")
+    # /externalDocs
+    merge_l1!(merged_spec, spec, "externalDocs")
+
+    # /servers
+    merge_array!(merged_spec, spec, "servers")
+    # /tags
+    merge_array!(merged_spec, spec, "tags")
+
+    # /paths/{path}/{method}
+    merge_l2(merged_spec, spec, "paths")
+
+    # /components/schemas/{schema}
+    # /components/responses/{response}
+    # /components/parameters/{parameter}
+    # /components/securitySchemes/{securityScheme}
+    merge_l2(merged_spec, spec, "components")
+
+    # /definitions/{definition}
+    merge_l1!(merged_spec, spec, "definitions")
+    # /securityDefinitions/{securityDefinition}
+    merge_l1!(merged_spec, spec, "securityDefinitions")
+    # /security (merge arrays)
+    merge_array!(merged_spec, spec, "security")
+
+    return merged_spec
+end
+
+function create_merged_spec(api_server::String=DEFAULT_API_SERVER)
+    paths = discover_apis(api_server)
+    merged_spec = Dict{String,Any}()
+    for name in keys(paths)
+        @info("merging $name")
+        url = string(api_server, paths[name]["serverRelativeURL"])
+        api_spec = fetch_json(url)
+        merge_spec!(merged_spec, api_spec)
+    end
+    if !haskey(merged_spec, "openapi")
+        merged_spec["openapi"] = "3.0.0"
+    end
+    return merged_spec
+end
+
+"""
+    get_specification(; outfile::String="k8s.json", api_server::String=DEFAULT_API_SERVER)
+
+Consolidate all discovered OpenAPI endpoint specifications and write them to a file.
+
+Keyword arguments:
+- `outfile::String`: The file to write the OpenAPI specification to. Defaults to `k8s.json`.
+- `api_server::String`: The URL of the k8s API server. Defaults to `http://localhost:8001`.
+"""
+function get_specification(; outfile::String="k8s.json", api_server::String=DEFAULT_API_SERVER)
+    merged_spec = create_merged_spec(api_server)
+    open(outfile, "w") do f
+        JSON.print(f, merged_spec, 2)
+    end
+end
+
+"""
+    merge_specifications(folder::String; outfile::String="k8s.json")
+
+Merge all JSON files in a given folder into a single OpenAPI specification.
+
+Arguments:
+- `folder::String`: The folder to read the JSON files from.
+
+Keyword arguments:
+- `outfile::String`: The file to write the OpenAPI specification to. Defaults to `k8s.json`.
+"""
+function merge_specifications(folder::String; outfile::String="k8s.json")
+    merged_spec = Dict{String,Any}()
+    for file in readdir(folder)
+        if endswith(file, ".json")
+            spec = JSON.parsefile(joinpath(folder, file))
+        elseif endswith(file, ".yaml")
+            spec = YAML.load_file(joinpath(folder, file))
+        else
+            continue
+        end
+        @info("merging $file")
+        merge_spec!(merged_spec, convert(Dict{String,Any}, spec))
+    end
+    if !haskey(merged_spec, "openapi")
+        merged_spec["openapi"] = "3.0.0"
+    end
+    open(outfile, "w") do f
+        JSON.print(f, merged_spec, 2)
+    end
+end
+
+"""
+    show_api_versions(specification::Dict{String,Any})
+
+Display all discovered API versions in a given OpenAPI specification.
+
+Arguments:
+- `specification::Dict{String,Any}`: The OpenAPI specification to read the API versions from.
+"""
+function show_api_versions(specification::Dict{String,Any})
+    tags = Set{String}()
+    for path in keys(specification["paths"])
+        for op in keys(specification["paths"][path])
+            if !isa(specification["paths"][path][op], Dict)
+                # @warn("possible invalid specification: /paths/$path/$op")
+                continue
+            end
+            if haskey(specification["paths"][path][op], "tags")
+                for tag in specification["paths"][path][op]["tags"]
+                    push!(tags, tag)
+                end
+            end
+        end
+    end
+    sorted_tags = sort(collect(tags))
+    for tag in sorted_tags
+        println(tag)
+    end
+end
+
+"""
+    show_api_versions(; specification_file::String="k8s.json")
+
+Display all discovered API versions in a generated `k8s.json` file.
+
+Keyword arguments:
+- `specification_file::String`: The JSON file to read the OpenAPI specification from. Defaults to `k8s.json`.
+"""
+function show_api_versions(; specification_file::String="k8s.json")
+    specification = JSON.parsefile(specification_file)
+    show_api_versions(specification)
+end
+
+end # module K8sOpenAPISpec

gen/README.md

@@ -2,7 +2,7 @@
 
 Use the bundled `generate.sh` script to generate OpenAPI client implementations.
 
-- Fetch the OpenAPI v2 (Swagger) specification from the `/openapi/v2` endpoint of a running k8s api server
+- Fetch the OpenAPI v2 (Swagger) specification from the `/openapi/v2` endpoint of a running k8s api server. For K8s servers that serve only the OpenAPI v3 specifications, use the included [K8sOpenAPISpec.jl](K8sOpenAPISpec.jl) script to consolidate specs from the API discovery endpoints into a single specfification file.
     - Ensure the k8s server has all the required CRDs installed
     - The specification file must be named `swagger.json`. It can be stored in any location, but store it in the `spec` folder if you wish to update the Kuber.jl package itself
 - The k8s OpenAPI spec uses a custom `int-or-string` format, that needs to be tweaked in the specification to be able to generate it correctly (see: https://github.com/kubernetes/kube-openapi/issues/52)
@@ -17,3 +17,43 @@ Use the bundled `generate.sh` script to generate OpenAPI client implementations.
 - Note:
     - the `api` folder in the output path will be renamed to `api_bak`
     - existing `api_bak` folder if any in output folder will be deleted
+
+## List Supported API Versions
+
+The included [K8sOpenAPISpec.jl](K8sOpenAPISpec.jl) script can be used to list the supported APIs and their versions from a given K8s OpenAPI specification.
+
+## K8sOpenAPISpec
+
+A standalone tool included to help in generating the API OpenAPI client from Kubernetes OpenAPI specifications.
+
+### `get_specification(; outfile::String="k8s.json", api_server::String=DEFAULT_API_SERVER)`
+
+Consolidate all discovered OpenAPI endpoint specifications and write them to a file.
+
+Keyword arguments:
+- `outfile::String`: The file to write the OpenAPI specification to. Defaults to `k8s.json`.
+- `api_server::String`: The URL of the k8s API server. Defaults to `http://localhost:8001`.
+
+### `merge_specifications(folder::String; outfile::String="k8s.json")`
+
+Merge all JSON files in a given folder into a single OpenAPI specification.
+
+Arguments:
+- `folder::String`: The folder to read the JSON files from.
+
+Keyword arguments:
+- `outfile::String`: The file to write the OpenAPI specification to. Defaults to `k8s.json`.
+
+### `show_api_versions(; specification_file::String="k8s.json")`
+
+Display all discovered API versions in a generated `k8s.json` file.
+
+Keyword arguments:
+- `specification_file::String`: The JSON file to read the OpenAPI specification from. Defaults to `k8s.json`.
+
+### show_api_versions(specification::Dict{String,Any})
+
+Display all discovered API versions in a given OpenAPI specification.
+
+Arguments:
+- `specification::Dict{String,Any}`: The OpenAPI specification to read the API versions from.

src/helpers.jl

@@ -431,7 +431,8 @@ function build_model_api_map(ctx::KuberContext)
         for name in names(types; all=true)
             (name in [:eval, Symbol("#eval"), :include, Symbol("#include"), Symbol(split(string(types), '.')[end])]) && continue
             # de-prioritize extensions for the default simpleapi mapping (so if a model already has a dedicated api version, do not use extensions)
-            haskey(modelapi, name) && (types === apimodule(ctx).Typedefs.ExtensionsV1beta1) && continue
+            # extensions are deprecated and not supported in k8s versions after v1.16
+            # haskey(modelapi, name) && (types === apimodule(ctx).Typedefs.ExtensionsV1beta1) && continue
             modelapi[name] = apiver
         end
     end