Skip to content

Commit

Permalink
Merge pull request #276 from PDOK/support-3d-geoms
Browse files Browse the repository at this point in the history
feat: support 3D geoms, by switching from geospatial library
  • Loading branch information
rkettelerij authored Jan 31, 2025
2 parents a44aaae + ed372e6 commit 74e80fd
Show file tree
Hide file tree
Showing 28 changed files with 27,674 additions and 53 deletions.
File renamed without changes.
10 changes: 10 additions & 0 deletions examples/config_all.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,16 @@ ogcApi:
urlAsHyperlink: true
- id: addresses2
tableName: addresses
metadata:
title: Addresses II
description: These are also example addresses
extent:
bbox:
- 50.2129
- 2.52713
- 55.7212
- 7.37403
storageCrs: http://www.opengis.net/def/crs/OGC/1.3/CRS84

3dgeovolumes:
tileServer: https://api.pdok.nl/kadaster/3d-basisvoorziening/ogc/v1/collections
Expand Down
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ require (
github.com/go-chi/chi/v5 v5.1.0
github.com/go-chi/cors v1.2.1
github.com/go-playground/validator/v10 v10.23.0
github.com/go-spatial/geom v0.1.0
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572
github.com/goccy/go-json v0.10.3
github.com/gomarkdown/markdown v0.0.0-20241105142532-d03b89096d81
Expand All @@ -24,6 +23,7 @@ require (
github.com/nicksnyder/go-i18n/v2 v2.4.1
github.com/qustavo/sqlhooks/v2 v2.1.0
github.com/stretchr/testify v1.9.0
github.com/twpayne/go-geom v1.6.0
github.com/urfave/cli/v2 v2.27.5
github.com/wk8/go-ordered-map/v2 v2.1.8
github.com/writeas/go-strip-markdown/v2 v2.1.1
Expand All @@ -43,7 +43,6 @@ require (
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.7 // indirect
github.com/gdey/errors v0.0.0-20190426172550-8ebd5bc891fb // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
Expand Down
14 changes: 10 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,14 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
github.com/PDOK/go-cloud-sqlite-vfs v0.3.0 h1:JTlXO1FzHEJXmGmdwK83q9LYPdWP63kr/beA2Ik1LBQ=
github.com/PDOK/go-cloud-sqlite-vfs v0.3.0/go.mod h1:+mZxO6New9AlVqFAF2rBEsOZB7J2aavwtdn3ifg021s=
github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA=
Expand All @@ -27,8 +33,6 @@ github.com/failsafe-go/failsafe-go v0.6.9 h1:7HWEzOlFOjNerxgWd8onWA2j/aEuqyAtuX6
github.com/failsafe-go/failsafe-go v0.6.9/go.mod h1:zb7xfp1/DJ7Mn4xJhVSZ9F2qmmMEGvYHxEOHYK5SIm0=
github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA=
github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU=
github.com/gdey/errors v0.0.0-20190426172550-8ebd5bc891fb h1:FYO+lZtAUnakgSW9xYs7QvgawjCDM5wgHaXoDhYHNH4=
github.com/gdey/errors v0.0.0-20190426172550-8ebd5bc891fb/go.mod h1:PFaV7MgSRe92Wo9O2H2i1CIm7urUk10AgdSHKyBfjmQ=
github.com/getkin/kin-openapi v0.128.0 h1:jqq3D9vC9pPq1dGcOCv7yOp1DaEe7c/T1vzcLbITSp4=
github.com/getkin/kin-openapi v0.128.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM=
github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
Expand All @@ -47,8 +51,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-spatial/geom v0.1.0 h1:IZ6LVz0Kkgd8+U83MmI0J4L4TBDzk5I8xiYe9Ci+aHk=
github.com/go-spatial/geom v0.1.0/go.mod h1:yNr22dnkQyFEwq2MJVmmiReiFesQpXKlosAG+qUSPus=
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
Expand All @@ -66,6 +68,8 @@ github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso=
github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
Expand Down Expand Up @@ -113,6 +117,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twpayne/go-geom v1.6.0 h1:WPOJLCdd8OdcnHvKQepLKwOZrn5BzVlNxtQB59IDHRE=
github.com/twpayne/go-geom v1.6.0/go.mod h1:Kr+Nly6BswFsKM5sd31YaoWS5PeDDH2NftJTK7Gd028=
github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w=
Expand Down
4 changes: 2 additions & 2 deletions internal/ogc/features/datasources/datasource.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (

"github.com/PDOK/gokoala/config"
"github.com/PDOK/gokoala/internal/ogc/features/domain"
"github.com/go-spatial/geom"
"github.com/twpayne/go-geom"
)

// Datasource holds all Features for a single object type in a specific projection/CRS.
Expand Down Expand Up @@ -48,7 +48,7 @@ type FeaturesCriteria struct {
OutputSRID int // derived from crs param when available, or WGS84 as default

// filtering by bounding box
Bbox *geom.Extent
Bbox *geom.Bounds

// filtering by reference date/time
TemporalCriteria TemporalCriteria
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func newLocalGeoPackage(gpkg *config.GeoPackageLocal) geoPackageBackend {
if gpkg.Download != nil {
downloadGeoPackage(gpkg)
}
conn := fmt.Sprintf("file:%s?mode=ro&_cache_size=%d", gpkg.File, gpkg.InMemoryCacheSize)
conn := fmt.Sprintf("file:%s?immutable=1&_cache_size=%d", gpkg.File, gpkg.InMemoryCacheSize)
db, err := sqlx.Open(sqliteDriverName, conn)
if err != nil {
log.Fatalf("failed to open GeoPackage: %v", err)
Expand Down
251 changes: 251 additions & 0 deletions internal/ogc/features/datasources/geopackage/encoding/geopackage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
// Package encoding based on https://github.com/go-spatial/geom/blob/master/encoding/gpkg/binary_header.go
//
// Copyright (c) 2017 go-spatial. Modified by PDOK.
// Licensed under the MIT license. See https://github.com/go-spatial/geom/blob/master/LICENSE for details.
package encoding

import (
"encoding/binary"
"errors"
"fmt"
"math"

"github.com/twpayne/go-geom"
"github.com/twpayne/go-geom/encoding/wkb"
"github.com/twpayne/go-geom/encoding/wkbcommon"
)

type EnvelopeType uint8

// Magic is the magic number encode in the header. It should be 0x4750
var Magic = [2]byte{0x47, 0x50}

// Decipher empty points with NaN as coordinates, in line with Requirement 152 of the spec (http://www.geopackage.org/spec/).
var gpkgNaNHandling = wkbcommon.WKBOptionEmptyPointHandling(wkbcommon.EmptyPointHandlingNaN)

const (
EnvelopeTypeNone = EnvelopeType(0)
EnvelopeTypeXY = EnvelopeType(1)
EnvelopeTypeXYZ = EnvelopeType(2)
EnvelopeTypeXYM = EnvelopeType(3)
EnvelopeTypeXYZM = EnvelopeType(4)
EnvelopeTypeInvalid = EnvelopeType(5)
)

// NumberOfElements that the particular Envelope Type will have.
func (et EnvelopeType) NumberOfElements() int {
switch et { //nolint:exhaustive
case EnvelopeTypeNone:
return 0
case EnvelopeTypeXY:
return 4
case EnvelopeTypeXYZ:
return 6
case EnvelopeTypeXYM:
return 6
case EnvelopeTypeXYZM:
return 8
default:
return -1
}
}

func (et EnvelopeType) String() string {
str := "NONEXYZMXYMINVALID"
switch et { //nolint:exhaustive
case EnvelopeTypeNone:
return str[0:4]
case EnvelopeTypeXY:
return str[4 : 4+2]
case EnvelopeTypeXYZ:
return str[4 : 4+3]
case EnvelopeTypeXYM:
return str[8 : 8+3]
case EnvelopeTypeXYZM:
return str[4 : 4+4]
default:
return str[11:]
}
}

// HEADER FLAG LAYOUT
// 7 6 5 4 3 2 1 0
// R R X Y E E E B
// R Reserved for future use. (should be set to 0)
// X GeoPackageBinary type // Normal or extented
// Y empty geometry
// E Envelope type
// B ByteOrder
// http://www.geopackage.org/spec/#flags_layout
const (
maskByteOrder = 1 << 0
maskEnvelopeType = 1<<3 | 1<<2 | 1<<1
maskEmptyGeometry = 1 << 4
maskGeoPackageBinary = 1 << 5
)

type headerFlags byte

func (hf headerFlags) String() string { return fmt.Sprintf("0x%02x", uint8(hf)) }

// Endian will return the encoded Endianess
func (hf headerFlags) Endian() binary.ByteOrder {
if hf&maskByteOrder == 0 {
return binary.BigEndian
}
return binary.LittleEndian
}

// Envelope returns the type of the envelope.
func (hf headerFlags) Envelope() EnvelopeType {
et := uint8((hf & maskEnvelopeType) >> 1)
if et >= uint8(EnvelopeTypeInvalid) {
return EnvelopeTypeInvalid
}
return EnvelopeType(et)
}

// IsEmpty returns whether or not the geometry is empty.
func (hf headerFlags) IsEmpty() bool { return ((hf & maskEmptyGeometry) >> 4) == 1 }

// IsStandard returns weather or not the geometry is a standard GeoPackage geometry type.
func (hf headerFlags) IsStandard() bool { return ((hf & maskGeoPackageBinary) >> 5) == 0 }

// BinaryHeader is the gpkg header that accompainies every feature.
type BinaryHeader struct {
// See: http://www.geopackage.org/spec/
magic [2]byte // should be 0x47 0x50 (GP in ASCII)
version uint8 // should be 0
flags headerFlags
srsid int32
envelope []float64
}

// decodeBinaryHeader decodes the data into the BinaryHeader
func decodeBinaryHeader(data []byte) (*BinaryHeader, error) {
if len(data) < 8 {
return nil, errors.New("not enough bytes")
}

var bh BinaryHeader
bh.magic[0] = data[0]
bh.magic[1] = data[1]
bh.version = data[2]
bh.flags = headerFlags(data[3])
en := bh.flags.Endian()
bh.srsid = int32(en.Uint32(data[4 : 4+4])) //nolint:gosec

bytes := data[8:]
et := bh.flags.Envelope()
if et == EnvelopeTypeInvalid {
return nil, errors.New("invalid envelope type")
}
if et == EnvelopeTypeNone {
return &bh, nil
}
num := et.NumberOfElements()
// there are 8 bytes per float64 value and we need num of them.
if len(bytes) < (num * 8) {
return nil, errors.New("not enough bytes")
}

bh.envelope = make([]float64, 0, num)
for i := 0; i < num; i++ {
bits := en.Uint64(bytes[i*8 : (i*8)+8])
bh.envelope = append(bh.envelope, math.Float64frombits(bits))
}
if bh.magic[0] != Magic[0] || bh.magic[1] != Magic[1] {
return &bh, errors.New("invalid magic number")
}
return &bh, nil

}

// Magic is the magic number encode in the header. It should be 0x4750
func (h *BinaryHeader) Magic() [2]byte {
if h == nil {
return Magic
}
return h.magic
}

// Version is the version number encode in the header.
func (h *BinaryHeader) Version() uint8 {
if h == nil {
return 0
}
return h.version
}

// EnvelopeType is the type of the envelope that is provided.
func (h *BinaryHeader) EnvelopeType() EnvelopeType {
if h == nil {
return EnvelopeTypeInvalid
}
return h.flags.Envelope()
}

// SRSID is the SRS id of the feature.
func (h *BinaryHeader) SRSID() int32 {
if h == nil {
return 0
}
return h.srsid
}

// Envelope is the bounding box of the feature, used for searching. If the EnvelopeType is EvelopeTypeNone, then there isn't a envelope encoded
// and a search without an index will need to be preformed. This is to save space.
func (h *BinaryHeader) Envelope() []float64 {
if h == nil {
return nil
}
return h.envelope
}

// IsGeometryEmpty tells us if the geometry should be considered empty.
func (h *BinaryHeader) IsGeometryEmpty() bool {
if h == nil {
return true
}
return h.flags.IsEmpty()
}

// IsStandardGeometry is the geometry a core/extended geometry type, or a user defined geometry type.
func (h *BinaryHeader) IsStandardGeometry() bool {
if h == nil {
return true
}
return h.flags.IsStandard()
}

// Size is the size of the header in bytes.
func (h *BinaryHeader) Size() int {
if h == nil {
return 0
}
return (len(h.envelope) * 8) + 8
}

// StandardBinary is the binary encoding plus some metadata
// should be stored as a blob
type StandardBinary struct {
Header *BinaryHeader
SRSID int32
Geometry geom.T
}

func DecodeGeometry(bytes []byte) (*StandardBinary, error) {
h, err := decodeBinaryHeader(bytes)
if err != nil {
return nil, err
}
geo, err := wkb.Unmarshal(bytes[h.Size():], gpkgNaNHandling)
if err != nil {
return nil, err
}
return &StandardBinary{
Header: h,
SRSID: h.SRSID(),
Geometry: geo,
}, nil
}
Loading

0 comments on commit 74e80fd

Please sign in to comment.