OGC API Features - Part 1 - Property filtering

.github/workflows/test.yml

@@ -26,7 +26,7 @@ jobs:
         run: go build -v ./...
 
       - name: Test
-        run: go test -v ./...
+        run: go test -v -shuffle=on ./...
 
       - name: Update coverage report
         uses: ncruces/go-coverage-report@v0

engine/config.go

@@ -37,10 +37,10 @@ func NewConfig(configFile string) *Config {
 		log.Fatalf("failed to unmarshal config file, error: %v", err)
 	}
 
-	return NewConfigFromStruct(config)
+	return NewConfigValid(config)
 }
 
-func NewConfigFromStruct(config *Config) *Config {
+func NewConfigValid(config *Config) *Config {
 	setDefaults(config)
 	validate(config)
 	return config
@@ -254,6 +254,15 @@ type CollectionEntryFeatures struct {
 
 	// Optional collection specific datasources. Mutually exclusive with top-level defined datasources.
 	Datasources *Datasources `yaml:"datasources"`
+
+	Filters struct {
+		// OAF Part 1: filter on feature properties
+		// https://docs.ogc.org/is/17-069r4/17-069r4.html#_parameters_for_filtering_on_feature_properties
+		Properties []PropertyFilter `yaml:"properties" validate:"dive"`
+
+		// OAF Part 3: add config for complex/CQL filters here
+		// placeholder
+	} `yaml:"filters"`
 }
 
 type CollectionEntryMaps struct {
@@ -282,7 +291,7 @@ type OgcAPIStyles struct {
 
 type OgcAPIFeatures struct {
 	Limit       Limit                 `yaml:"limit"`
-	Datasources *Datasources          `yaml:"datasources"`
+	Datasources *Datasources          `yaml:"datasources"` // optional since you can also define datasources at the collection level
 	Basemap     string                `yaml:"basemap" default:"OSM"`
 	Collections GeoSpatialCollections `yaml:"collections" validate:"required"`
 }
@@ -311,6 +320,15 @@ func (oaf OgcAPIFeatures) ProjectionsForCollection(collectionID string) []string
 	return result
 }
 
+func (oaf OgcAPIFeatures) PropertyFiltersForCollection(collectionID string) []PropertyFilter {
+	for _, coll := range oaf.Collections {
+		if coll.ID == collectionID && coll.Features != nil && coll.Features.Filters.Properties != nil {
+			return coll.Features.Filters.Properties
+		}
+	}
+	return []PropertyFilter{}
+}
+
 type OgcAPIMaps struct {
 	Collections GeoSpatialCollections `yaml:"collections"`
 }
@@ -394,6 +412,12 @@ type GeoPackageCloud struct {
 	Cache *string `yaml:"cache" validate:"omitempty,dir"`
 }
 
+type PropertyFilter struct {
+	// needs to match with a column name in the feature table (in the configured datasource)
+	Name        string `yaml:"name" validate:"required"`
+	Description string `yaml:"description" default:"Filter features by this property"`
+}
+
 type SupportedSrs struct {
 	Srs            string         `yaml:"srs" validate:"required,startswith=EPSG:"`
 	ZoomLevelRange ZoomLevelRange `yaml:"zoomLevelRange" validate:"required"`

engine/engine.go

@@ -54,7 +54,7 @@ func NewEngine(configFile string, openAPIFile string) *Engine {
 func NewEngineWithConfig(config *Config, openAPIFile string) *Engine {
 	contentNegotiation := newContentNegotiation(config.AvailableLanguages)
 	templates := newTemplates(config)
-	openAPI := newOpenAPI(config, openAPIFile)
+	openAPI := newOpenAPI(config, []string{openAPIFile}, nil)
 
 	engine := &Engine{
 		Config:    config,
@@ -100,7 +100,7 @@ func (e *Engine) startServer(name string, address string, shutdownDelay int, rou
 	defer stop()
 
 	go func() {
-		log.Printf("%s listening on %s", name, address)
+		log.Printf("%s listening on http://%2s", name, address)
 		// ListenAndServe always returns a non-nil error. After Shutdown or
 		// Close, the returned error is ErrServerClosed
 		if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
@@ -129,10 +129,18 @@ func (e *Engine) startServer(name string, address string, shutdownDelay int, rou
 	return server.Shutdown(timeoutCtx)
 }
 
+// RegisterShutdownHook register a func to execute during graceful shutdown, e.g. to clean up resources.
 func (e *Engine) RegisterShutdownHook(fn func()) {
 	e.shutdownHooks = append(e.shutdownHooks, fn)
 }
 
+// RebuildOpenAPI rebuild the full OpenAPI spec with the newly given parameters.
+// Use only once during bootstrap for specific use cases! For example: when you want to expand a
+// specific part of the OpenAPI spec with data outside the configuration file (e.g. from a database).
+func (e *Engine) RebuildOpenAPI(openAPIParams any) {
+	e.OpenAPI = newOpenAPI(e.Config, e.OpenAPI.extraOpenAPIFiles, openAPIParams)
+}
+
 // ParseTemplate parses both HTML and non-HTML templates depending on the format given in the TemplateKey and
 // stores it in the engine for future rendering using RenderAndServePage.
 func (e *Engine) ParseTemplate(key TemplateKey) {

engine/openapi.go

@@ -35,13 +35,15 @@ const (
 )
 
 type OpenAPI struct {
-	config   *Config
 	spec     *openapi3.T
 	SpecJSON []byte
-	router   routers.Router
+
+	config            *Config
+	router            routers.Router
+	extraOpenAPIFiles []string
 }
 
-func newOpenAPI(config *Config, openAPIFile string) *OpenAPI {
+func newOpenAPI(config *Config, extraOpenAPIFiles []string, openAPIParams any) *OpenAPI {
 	setupRequestResponseValidation()
 	ctx := context.Background()
 
@@ -62,27 +64,26 @@ func newOpenAPI(config *Config, openAPIFile string) *OpenAPI {
 	if config.OgcAPI.GeoVolumes != nil {
 		defaultOpenAPIFiles = append(defaultOpenAPIFiles, geoVolumesSpec)
 	}
+
 	// add preamble first
 	openAPIFiles := []string{preamble}
-	if openAPIFile != "" {
-		// add provided spec thereafter, to allow it to override defaults of following specs
-		openAPIFiles = append(openAPIFiles, openAPIFile)
-	}
+	// add extra spec(s) thereafter, to allow it to override default openapi specs
+	openAPIFiles = append(openAPIFiles, extraOpenAPIFiles...)
 	openAPIFiles = append(openAPIFiles, defaultOpenAPIFiles...)
 
-	resultSpec, resultSpecJSON := mergeSpecs(ctx, config, openAPIFiles)
+	resultSpec, resultSpecJSON := mergeSpecs(ctx, config, openAPIFiles, openAPIParams)
 	validateSpec(ctx, resultSpec, resultSpecJSON)
 
 	for _, server := range resultSpec.Servers {
 		server.URL = normalizeBaseURL(server.URL)
-		log.Printf("url used for OpenAPI validation: %v", server.URL)
 	}
 
 	return &OpenAPI{
-		config:   config,
-		spec:     resultSpec,
-		SpecJSON: util.PrettyPrintJSON(resultSpecJSON, ""),
-		router:   newOpenAPIRouter(resultSpec),
+		config:            config,
+		spec:              resultSpec,
+		SpecJSON:          util.PrettyPrintJSON(resultSpecJSON, ""),
+		router:            newOpenAPIRouter(resultSpec),
+		extraOpenAPIFiles: extraOpenAPIFiles,
 	}
 }
 
@@ -126,8 +127,8 @@ func setupRequestResponseValidation() {
 //
 // The OpenAPI spec optionally provided through the CLI should be the second (after preamble) item in the
 // `files` slice since it allows the user to override other/default specs.
-func mergeSpecs(ctx context.Context, config *Config, files []string) (*openapi3.T, []byte) {
-	loader := &openapi3.Loader{Context: ctx, IsExternalRefsAllowed: true}
+func mergeSpecs(ctx context.Context, config *Config, files []string, params any) (*openapi3.T, []byte) {
+	loader := &openapi3.Loader{Context: ctx, IsExternalRefsAllowed: false}
 
 	if len(files) < 1 {
 		log.Fatalf("files can't be empty, at least OGC Common is expected")
@@ -136,7 +137,10 @@ func mergeSpecs(ctx context.Context, config *Config, files []string) (*openapi3.
 	var resultSpec *openapi3.T
 
 	for _, file := range files {
-		specJSON := renderOpenAPITemplate(config, file)
+		if file == "" {
+			continue
+		}
+		specJSON := renderOpenAPITemplate(config, file, params)
 		var mergedJSON []byte
 		if resultSpecJSON == nil {
 			mergedJSON = specJSON
@@ -180,12 +184,12 @@ func newOpenAPIRouter(doc *openapi3.T) routers.Router {
 	return openAPIRouter
 }
 
-func renderOpenAPITemplate(config *Config, fileName string) []byte {
+func renderOpenAPITemplate(config *Config, fileName string, params any) []byte {
 	file := filepath.Clean(fileName)
 	parsed := texttemplate.Must(texttemplate.New(filepath.Base(file)).Funcs(globalTemplateFuncs).ParseFiles(file))
 
 	var rendered bytes.Buffer
-	if err := parsed.Execute(&rendered, &TemplateData{Config: config}); err != nil {
+	if err := parsed.Execute(&rendered, &TemplateData{Config: config, Params: params}); err != nil {
 		log.Fatalf("failed to render %s, error: %v", file, err)
 	}
 	return rendered.Bytes()
@@ -243,8 +247,8 @@ func (o *OpenAPI) getRequestValidationInput(r *http.Request) (*openapi3filter.Re
 // requests against the OpenAPI spec. This involves:
 //
 //   - striping the context root (path) from the base URL. If you use a context root we expect
-//     you have a proxying fronting GoKoala it from requests, therefore we also need to strip it from
-//     the base URL used during OpenAPI validation
+//     you to have a proxy fronting GoKoala, therefore we also  need to strip it from the base
+//     URL used during OpenAPI validation
 //
 //   - replacing HTTPS scheme with HTTP. Since GoKoala doesn't support HTTPS we always perform
 //     OpenAPI validation against HTTP requests. Note: it's possible to offer GoKoala over HTTPS, but you'll

engine/openapi_test.go

@@ -198,7 +198,7 @@ func Test_newOpenAPI(t *testing.T) {
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			openAPI := newOpenAPI(test.args.config, test.args.openAPIFile)
+			openAPI := newOpenAPI(test.args.config, []string{test.args.openAPIFile}, nil)
 			assert.NotNil(t, openAPI)
 
 			// verify resulting OpenAPI spec contains expected strings (keywords, paths, etc)

engine/template.go

@@ -33,6 +33,16 @@ var (
 	globalTemplateFuncs texttemplate.FuncMap
 )
 
+func init() {
+	customFuncs := texttemplate.FuncMap{
+		// custom template functions
+		"markdown":   markdown,
+		"unmarkdown": unmarkdown,
+	}
+	sprigFuncs := sprig.FuncMap() // we also support https://github.com/go-task/slim-sprig functions
+	globalTemplateFuncs = combineFuncMaps(customFuncs, sprigFuncs)
+}
+
 // TemplateKey unique key to register and lookup Go templates
 type TemplateKey struct {
 	// Name of the template, the filename including extension
@@ -148,14 +158,6 @@ func newTemplates(config *Config) *Templates {
 		config:            config,
 		localizers:        newLocalizers(config.AvailableLanguages),
 	}
-	customFuncs := texttemplate.FuncMap{
-		// custom template functions
-		"markdown":   markdown,
-		"unmarkdown": unmarkdown,
-	}
-	// we also support https://github.com/go-task/slim-sprig functions
-	sprigFuncs := sprig.FuncMap()
-	globalTemplateFuncs = combineFuncMaps(customFuncs, sprigFuncs)
 	return templates
 }
 

engine/templates/openapi/3dgeovolumes.go.json

@@ -12,15 +12,15 @@
   },
   "tags" : [ ],
   "paths" : {
-    {{- range $index, $type := .Config.OgcAPI.GeoVolumes.Collections -}}
-    {{ if and $type.GeoVolumes $type.GeoVolumes.URITemplate3dTiles }}
+    {{- range $index, $coll := .Config.OgcAPI.GeoVolumes.Collections -}}
+    {{ if and $coll.GeoVolumes $coll.GeoVolumes.URITemplate3dTiles }}
     {{- if $index -}},{{- end -}}
-    "/collections/{{ $type.ID }}/3dtiles" : {
+    "/collections/{{ $coll.ID }}/3dtiles" : {
       "get" : {
         "tags" : [ "3D Tiles" ],
-        "summary" : "retrieve the root 3D Tiles tileset of the feature collection '{{ $type.ID }}'",
+        "summary" : "retrieve the root 3D Tiles tileset of the feature collection '{{ $coll.ID }}'",
         "description" : "Access a 3D Tiles 1.1 tileset with implicit quadtree tiling.",
-        "operationId" : "get3dTileset.{{ $type.ID }}",
+        "operationId" : "get3dTileset.{{ $coll.ID }}",
         "parameters" : [ ],
         "responses" : {
           "200" : {
@@ -42,12 +42,12 @@
         }
       }
     },
-    "/collections/{{ $type.ID }}/3dtiles/{{ $type.GeoVolumes.URITemplate3dTiles }}" : {
+    "/collections/{{ $coll.ID }}/3dtiles/{{ $coll.GeoVolumes.URITemplate3dTiles }}" : {
       "get" : {
         "tags" : [ "3D Tiles" ],
-        "summary" : "retrieve a glTF tile of the feature collection '{{ $type.ID }}'",
+        "summary" : "retrieve a glTF tile of the feature collection '{{ $coll.ID }}'",
         "description" : "Access a 3D Tiles 1.1 Content file, a glTF 2.0 binary file.",
-        "operationId" : "get3dTilesContent.{{ $type.ID }}",
+        "operationId" : "get3dTilesContent.{{ $coll.ID }}",
         "parameters" : [ {
           "$ref" : "#/components/parameters/level"
         }, {
@@ -76,14 +76,14 @@
       }
     }
     {{ end }}
-    {{ if and $type.GeoVolumes $type.GeoVolumes.URITemplateImplicitTilingSubtree }}
+    {{ if and $coll.GeoVolumes $coll.GeoVolumes.URITemplateImplicitTilingSubtree }}
     ,
-    "/collections/{{ $type.ID }}/3dtiles/{{ $type.GeoVolumes.URITemplateImplicitTilingSubtree }}" : {
+    "/collections/{{ $coll.ID }}/3dtiles/{{ $coll.GeoVolumes.URITemplateImplicitTilingSubtree }}" : {
       "get" : {
         "tags" : [ "3D Tiles" ],
-        "summary" : "retrieve a 3D Tiles subtree of the feature collection '{{ $type.ID }}'",
+        "summary" : "retrieve a 3D Tiles subtree of the feature collection '{{ $coll.ID }}'",
         "description" : "Access a 3D Tiles 1.1 Subtree file.",
-        "operationId" : "get3dTilesSubtree.{{ $type.ID }}",
+        "operationId" : "get3dTilesSubtree.{{ $coll.ID }}",
         "parameters" : [ {
           "$ref" : "#/components/parameters/level"
         }, {
@@ -112,14 +112,14 @@
       }
     }
     {{ end }}
-    {{ if and $type.GeoVolumes $type.GeoVolumes.HasDTM }}
+    {{ if and $coll.GeoVolumes $coll.GeoVolumes.HasDTM }}
     ,
-    "/collections/{{ $type.ID }}/quantized-mesh/{{ $type.GeoVolumes.URITemplateDTM }}" : {
+    "/collections/{{ $coll.ID }}/quantized-mesh/{{ $coll.GeoVolumes.URITemplateDTM }}" : {
       "get" : {
         "tags" : [ "3D Tiles" ],
         "summary" : "retrieve digital terrain model (DTM)",
         "description" : "Access the digital terrain model (DTM) in Quantized Mesh format.",
-        "operationId" : "getDTM.{{ $type.ID }}",
+        "operationId" : "getDTM.{{ $coll.ID }}",
         "parameters" : [ {
           "$ref" : "#/components/parameters/level"
         }, {

engine/templates/openapi/common-collections.go.json

@@ -76,16 +76,16 @@
         }
       }
     },
-    {{- range $index, $type := .Config.AllCollections -}}
+    {{- range $index, $coll := .Config.AllCollections -}}
     {{- if $index -}},{{- end -}}
-    "/collections/{{ $type.ID }}": {
+    "/collections/{{ $coll.ID }}": {
       "get": {
         "tags": [
           "Collections"
         ],
-        "summary": "describes a {{ $type.ID }} collection",
-        "description": "{{ $type.ID }} collection (geospatial data resource) in this dataset.",
-        "operationId": "getCollection.{{ $type.ID }}",
+        "summary": "describes a {{ $coll.ID }} collection",
+        "description": "{{ $coll.ID }} collection (geospatial data resource) in this dataset.",
+        "operationId": "getCollection.{{ $coll.ID }}",
         "parameters": [
           {
             "name": "f",