Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
sql, geo: add support for writing geospatial inverted indexes.
Browse files Browse the repository at this point in the history
Release note: None
sumeerbhola committed Apr 13, 2020
1 parent 4270810 commit 89465d0
Showing 16 changed files with 790 additions and 355 deletions.
4 changes: 4 additions & 0 deletions docs/generated/sql/functions.md
Original file line number Diff line number Diff line change
@@ -1163,6 +1163,10 @@ SELECT * FROM crdb_internal.check_consistency(true, ‘\x02’, ‘\x04’)</p>
</span></td></tr>
<tr><td><a name="crdb_internal.num_inverted_index_entries"></a><code>crdb_internal.num_inverted_index_entries(val: anyelement[]) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
</span></td></tr>
<tr><td><a name="crdb_internal.num_inverted_index_entries"></a><code>crdb_internal.num_inverted_index_entries(val: geography) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
</span></td></tr>
<tr><td><a name="crdb_internal.num_inverted_index_entries"></a><code>crdb_internal.num_inverted_index_entries(val: geometry) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
</span></td></tr>
<tr><td><a name="crdb_internal.num_inverted_index_entries"></a><code>crdb_internal.num_inverted_index_entries(val: jsonb) &rarr; <a href="int.html">int</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
</span></td></tr>
<tr><td><a name="crdb_internal.pretty_key"></a><code>crdb_internal.pretty_key(raw_key: <a href="bytes.html">bytes</a>, skip_fields: <a href="int.html">int</a>) &rarr; <a href="string.html">string</a></code></td><td><span class="funcdesc"><p>This function is used only by CockroachDB’s developers for testing purposes.</p>
19 changes: 14 additions & 5 deletions pkg/cli/testdata/dump/inverted_index
Original file line number Diff line number Diff line change
@@ -5,14 +5,19 @@ CREATE TABLE d.t (
b JSON,
c INT[],
d INT[],
e geography(geometry, 4326),
f geometry(point, 4326),

INVERTED INDEX idx (a),
INVERTED INDEX idx3 (c)
INVERTED INDEX idx3 (c),
INVERTED INDEX idx5 (e)
);

CREATE INVERTED INDEX idx2 ON d.t (b);
CREATE INVERTED INDEX idx4 ON d.t (d);
CREATE INVERTED INDEX idx6 ON d.t (f);

INSERT INTO d.t VALUES ('{"a": "b"}', '{"c": "d"}', ARRAY[1], ARRAY[2]);
INSERT INTO d.t VALUES ('{"a": "b"}', '{"c": "d"}', ARRAY[1], ARRAY[2], 'POINT(1.0 1.0)', 'POINT(1.0 1.0)');
----
INSERT 1

@@ -24,14 +29,18 @@ CREATE TABLE t (
b JSONB NULL,
c INT8[] NULL,
d INT8[] NULL,
e GEOGRAPHY(GEOMETRY,4326) NULL,
f GEOMETRY(POINT,4326) NULL,
INVERTED INDEX idx (a),
INVERTED INDEX idx3 (c),
INVERTED INDEX idx5 (e),
INVERTED INDEX idx2 (b),
INVERTED INDEX idx4 (d),
FAMILY "primary" (a, b, c, d, rowid)
INVERTED INDEX idx6 (f),
FAMILY "primary" (a, b, c, d, e, f, rowid)
);

INSERT INTO t (a, b, c, d) VALUES
('{"a": "b"}', '{"c": "d"}', ARRAY[1], ARRAY[2]);
INSERT INTO t (a, b, c, d, e, f) VALUES
('{"a": "b"}', '{"c": "d"}', ARRAY[1], ARRAY[2], '0101000000000000000000F03F000000000000F03F', '0101000000000000000000F03F000000000000F03F');
----
----
181 changes: 151 additions & 30 deletions pkg/geo/geoindex/config.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions pkg/geo/geoindex/config.proto
Original file line number Diff line number Diff line change
@@ -19,6 +19,7 @@ import "gogoproto/gogo.proto";
//
// At the moment, only one major indexing strategy is implemented (S2 cells).
message Config {
option (gogoproto.equal) = true;
option (gogoproto.onlyone) = true;
S2GeographyConfig s2_geography = 1;
S2GeometryConfig s2_geometry = 2;
@@ -31,6 +32,7 @@ message Config {
// TODO(sumeer): Based on experiments, reduce the knobs below by making the
// covering self-tuning.
message S2Config {
option (gogoproto.equal) = true;
// MinLevel is the minimum cell level stored in the index. If left unset, it
// defaults to 0.
int32 min_level = 1;
@@ -46,10 +48,12 @@ message S2Config {
}

message S2GeographyConfig {
option (gogoproto.equal) = true;
S2Config s2_config = 1;
}

message S2GeometryConfig {
option (gogoproto.equal) = true;
// The rectangle bounds of the plane that will be efficiently indexed. Shapes
// should rarely exceed these bounds.
double min_x = 1;
16 changes: 16 additions & 0 deletions pkg/geo/geoindex/index.go
Original file line number Diff line number Diff line change
@@ -118,6 +118,13 @@ type GeometryIndex interface {
innerCovering(g *geo.Geometry) s2.CellUnion
}

func IsEmptyConfig(cfg *Config) bool {
if cfg == nil {
return true
}
return cfg.S2Geography == nil && cfg.S2Geometry == nil
}

// Key is one entry under which a geospatial shape is stored on behalf of an
// Index. The index is of the form (Key, Primary Key).
type Key uint64
@@ -481,3 +488,12 @@ func generateRPExprForTree(rootID s2.CellID, presentCells map[s2.CellID]struct{}
}
return expr
}

func defaultS2Config() *S2Config {
return &S2Config{
MinLevel: 0,
MaxLevel: 30,
LevelMod: 1,
MaxCells: 1,
}
}
6 changes: 6 additions & 0 deletions pkg/geo/geoindex/s2_gg_index.go
Original file line number Diff line number Diff line change
@@ -42,6 +42,12 @@ func NewS2GGIndex(cfg S2GeographyConfig) GeographyIndex {
}
}

func DefaultGeographyIndexConfig() *Config {
return &Config{
S2Geography: &S2GeographyConfig{S2Config: defaultS2Config()},
}
}

// InvertedIndexKeys implements the GeographyIndex interface.
func (i *s2GGIndex) InvertedIndexKeys(c context.Context, g *geo.Geography) ([]Key, error) {
r, err := g.AsS2()
13 changes: 13 additions & 0 deletions pkg/geo/geoindex/s2_gm_index.go
Original file line number Diff line number Diff line change
@@ -51,6 +51,19 @@ func NewS2GMIndex(cfg S2GeometryConfig) GeometryIndex {
}
}

func DefaultGeometryIndexConfig() *Config {
return &Config{
S2Geometry: &S2GeometryConfig{
// Arbitrary bounding box.
// TODO(sumeer): replace with parameters specified by CREATE INDEX.
MinX: 0,
MaxX: 10000,
MinY: 0,
MaxY: 10000,
S2Config: defaultS2Config()},
}
}

// A cell id unused by S2. We use it to index geometries that exceed the
// configured bounds.
const exceedsBoundsCellID = s2.CellID(^uint64(0))
12 changes: 12 additions & 0 deletions pkg/sql/create_index.go
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@ import (
"context"

"github.com/cockroachdb/cockroach/pkg/clusterversion"
"github.com/cockroachdb/cockroach/pkg/geo/geoindex"
"github.com/cockroachdb/cockroach/pkg/server/telemetry"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgcode"
@@ -22,6 +23,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
"github.com/cockroachdb/cockroach/pkg/sql/sqltelemetry"
"github.com/cockroachdb/cockroach/pkg/sql/types"
"github.com/cockroachdb/errors"
)

@@ -147,6 +149,16 @@ func MakeIndexDescriptor(
return nil, pgerror.New(pgcode.InvalidSQLStatementName, "inverted indexes can't be unique")
}
indexDesc.Type = sqlbase.IndexDescriptor_INVERTED
columnDesc, _, err := tableDesc.FindColumnByName(n.Columns[0].Column)
if err != nil {
return nil, err
}
if columnDesc.Type.InternalType.Family == types.GeometryFamily {
indexDesc.GeoConfig = *geoindex.DefaultGeometryIndexConfig()
}
if columnDesc.Type.InternalType.Family == types.GeographyFamily {
indexDesc.GeoConfig = *geoindex.DefaultGeographyIndexConfig()
}
telemetry.Inc(sqltelemetry.InvertedIndexCounter)
}

Loading

0 comments on commit 89465d0

Please sign in to comment.