From 507a2be8e4fda152d517dcb972be6353a6da2914 Mon Sep 17 00:00:00 2001 From: shollyman Date: Thu, 16 Jun 2022 11:28:58 -0700 Subject: [PATCH] feat(bigquery): support partial projection of table metadata (#6186) * feat(bigquery): support partial projection of table metadata --- bigquery/go.mod | 2 +- bigquery/go.sum | 3 +- bigquery/table.go | 60 +++++++++++++++++++++++++----- bigquery/table_integration_test.go | 35 +++++++++++++++++ 4 files changed, 88 insertions(+), 12 deletions(-) diff --git a/bigquery/go.mod b/bigquery/go.mod index 6c9350d69d5a..3ae9064bce4c 100644 --- a/bigquery/go.mod +++ b/bigquery/go.mod @@ -14,7 +14,7 @@ require ( golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f google.golang.org/api v0.84.0 - google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 + google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.0 ) diff --git a/bigquery/go.sum b/bigquery/go.sum index 14cf40fa964c..dcf3f274971c 100644 --- a/bigquery/go.sum +++ b/bigquery/go.sum @@ -588,9 +588,8 @@ google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac h1:ByeiW1F67iV9o8ipGskA+HWzSkMbRJuKLlwCdPxzn7A= google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 h1:4SPz2GL2CXJt28MTF8V6Ap/9ZiVbQlJeGSd9qtA7DLs= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/bigquery/table.go b/bigquery/table.go index abf5aa69deaf..5ff66f39f215 100644 --- a/bigquery/table.go +++ b/bigquery/table.go @@ -664,22 +664,64 @@ func (tm *TableMetadata) toBQ() (*bq.Table, error) { return t, nil } +// We use this for the option pattern rather than exposing the underlying +// discovery type directly. +type tableGetCall struct { + call *bq.TablesGetCall +} + +// TableMetadataOption allow requests to alter requests for table metadata. +type TableMetadataOption func(*tableGetCall) + +// TableMetadataView specifies which details about a table are desired. +type TableMetadataView string + +const ( + // BasicMetadataView populates basic table information including schema partitioning, + // but does not contain storage statistics like number or rows or bytes. This is a more + // efficient view to use for large tables or higher metadata query rates. + BasicMetadataView TableMetadataView = "BASIC" + + // FullMetadataView returns all table information, including storage statistics. It currently + // returns the same information as StorageStatsMetadataView, but may include additional information + // in the future. + FullMetadataView TableMetadataView = "FULL" + + // StorageStatsMetadataView includes all information from the basic view, and includes storage statistics. It currently + StorageStatsMetadataView TableMetadataView = "STORAGE_STATS" +) + +// WithMetadataView is used to customize what details are returned when interrogating a +// table via the Metadata() call. Generally this is used to limit data returned for performance +// reasons (such as large tables that take time computing storage statistics). +func WithMetadataView(tmv TableMetadataView) TableMetadataOption { + return func(tgc *tableGetCall) { + tgc.call.View(string(tmv)) + } +} + // Metadata fetches the metadata for the table. -func (t *Table) Metadata(ctx context.Context) (md *TableMetadata, err error) { +func (t *Table) Metadata(ctx context.Context, opts ...TableMetadataOption) (md *TableMetadata, err error) { ctx = trace.StartSpan(ctx, "cloud.google.com/go/bigquery.Table.Metadata") defer func() { trace.EndSpan(ctx, err) }() - req := t.c.bqs.Tables.Get(t.ProjectID, t.DatasetID, t.TableID).Context(ctx) - setClientHeader(req.Header()) - var table *bq.Table - err = runWithRetry(ctx, func() (err error) { - table, err = req.Do() + tgc := &tableGetCall{ + call: t.c.bqs.Tables.Get(t.ProjectID, t.DatasetID, t.TableID).Context(ctx), + } + + for _, o := range opts { + o(tgc) + } + + setClientHeader(tgc.call.Header()) + var res *bq.Table + if err := runWithRetry(ctx, func() (err error) { + res, err = tgc.call.Do() return err - }) - if err != nil { + }); err != nil { return nil, err } - return bqToTableMetadata(table, t.c) + return bqToTableMetadata(res, t.c) } func bqToTableMetadata(t *bq.Table, c *Client) (*TableMetadata, error) { diff --git a/bigquery/table_integration_test.go b/bigquery/table_integration_test.go index 86b046b1be36..c6e06fd1223a 100644 --- a/bigquery/table_integration_test.go +++ b/bigquery/table_integration_test.go @@ -237,7 +237,42 @@ func TestIntegration_TableMetadata(t *testing.T) { } } } +} + +func TestIntegration_TableMetadataOptions(t *testing.T) { + if client == nil { + t.Skip("Integration tests skipped") + } + ctx := context.Background() + testTable := dataset.Table(tableIDs.New()) + id, _ := testTable.Identifier(StandardSQLID) + sql := "CREATE TABLE %s AS SELECT num FROM UNNEST(GENERATE_ARRAY(0,5)) as num" + q := client.Query(fmt.Sprintf(sql, id)) + if _, err := q.Read(ctx); err != nil { + t.Fatalf("failed to create table: %v", err) + } + defaultMeta, err := testTable.Metadata(ctx) + if err != nil { + t.Fatalf("failed to get default metadata: %v", err) + } + if defaultMeta.NumBytes <= 0 { + t.Errorf("expected default positive NumBytes, got %d", defaultMeta.NumBytes) + } + if defaultMeta.LastModifiedTime.IsZero() { + t.Error("expected default LastModifiedTime to be populated, is zero value") + } + // Specify a subset of metadata. + basicMeta, err := testTable.Metadata(ctx, WithMetadataView(BasicMetadataView)) + if err != nil { + t.Fatalf("failed to get basic metadata: %v", err) + } + if basicMeta.NumBytes != 0 { + t.Errorf("expected basic NumBytes to be zero, got %d", defaultMeta.NumBytes) + } + if !basicMeta.LastModifiedTime.IsZero() { + t.Errorf("expected basic LastModifiedTime to be zero, is %v", basicMeta.LastModifiedTime) + } } func TestIntegration_TableUpdateLabels(t *testing.T) {