diff --git a/.kokoro/presubmit/prerelease-deps-3.8.cfg b/.kokoro/presubmit/prerelease-deps-3.8.cfg index f06806baf..fabe3e347 100644 --- a/.kokoro/presubmit/prerelease-deps-3.8.cfg +++ b/.kokoro/presubmit/prerelease-deps-3.8.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "prerelease_deps" -} \ No newline at end of file + value: "prerelease_deps-3.8" +} diff --git a/.kokoro/presubmit/snippets-3.10.cfg b/.kokoro/presubmit/snippets-3.10.cfg new file mode 100644 index 000000000..dde182fb9 --- /dev/null +++ b/.kokoro/presubmit/snippets-3.10.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "snippets-3.10" +} diff --git a/.kokoro/presubmit/snippets-2.7.cfg b/.kokoro/presubmit/system-3.10.cfg similarity index 82% rename from .kokoro/presubmit/snippets-2.7.cfg rename to .kokoro/presubmit/system-3.10.cfg index 3bd6134d2..30956a3ab 100644 --- a/.kokoro/presubmit/snippets-2.7.cfg +++ b/.kokoro/presubmit/system-3.10.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "snippets-2.7" + value: "system-3.10" } diff --git a/CHANGELOG.md b/CHANGELOG.md index d15f22851..4e10ad826 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,54 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +### [2.30.1](https://www.github.com/googleapis/python-bigquery/compare/v2.30.0...v2.30.1) (2021-11-04) + + +### Bug Fixes + +* error if eval()-ing repr(SchemaField) ([#1046](https://www.github.com/googleapis/python-bigquery/issues/1046)) ([13ac860](https://www.github.com/googleapis/python-bigquery/commit/13ac860de689ea13b35932c67042bc35e388cb30)) + + +### Documentation + +* show gcloud command to authorize against sheets ([#1045](https://www.github.com/googleapis/python-bigquery/issues/1045)) ([20c9024](https://www.github.com/googleapis/python-bigquery/commit/20c9024b5760f7ae41301f4da54568496922cbe2)) +* use stable URL for pandas intersphinx links ([#1048](https://www.github.com/googleapis/python-bigquery/issues/1048)) ([73312f8](https://www.github.com/googleapis/python-bigquery/commit/73312f8f0f22ff9175a4f5f7db9bb438a496c164)) + +## [2.30.0](https://www.github.com/googleapis/python-bigquery/compare/v2.29.0...v2.30.0) (2021-11-03) + + +### Features + +* accept TableListItem where TableReference is accepted ([#1016](https://www.github.com/googleapis/python-bigquery/issues/1016)) ([fe16adc](https://www.github.com/googleapis/python-bigquery/commit/fe16adc86a170d0992c32091b349b036f8b43884)) +* support Python 3.10 ([#1043](https://www.github.com/googleapis/python-bigquery/issues/1043)) ([5bbb832](https://www.github.com/googleapis/python-bigquery/commit/5bbb832a83ebb66db4b5ee740cdfc53f4df8430b)) + + +### Documentation + +* add code samples for Jupyter/IPython magics ([#1013](https://www.github.com/googleapis/python-bigquery/issues/1013)) ([61141ee](https://www.github.com/googleapis/python-bigquery/commit/61141ee0634024ad261d1595c95cd14a896fb87e)) +* **samples:** add create external table with hive partitioning ([#1033](https://www.github.com/googleapis/python-bigquery/issues/1033)) ([d64f5b6](https://www.github.com/googleapis/python-bigquery/commit/d64f5b682854a2293244426316890df4ab1e079e)) + +## [2.29.0](https://www.github.com/googleapis/python-bigquery/compare/v2.28.1...v2.29.0) (2021-10-27) + + +### Features + +* add `QueryJob.schema` property for dry run queries ([#1014](https://www.github.com/googleapis/python-bigquery/issues/1014)) ([2937fa1](https://www.github.com/googleapis/python-bigquery/commit/2937fa1386898766c561579fd39d42958182d260)) +* add session and connection properties to QueryJobConfig ([#1024](https://www.github.com/googleapis/python-bigquery/issues/1024)) ([e4c94f4](https://www.github.com/googleapis/python-bigquery/commit/e4c94f446c27eb474f30b033c1b62d11bd0acd98)) +* add support for INTERVAL data type to `list_rows` ([#840](https://www.github.com/googleapis/python-bigquery/issues/840)) ([e37380a](https://www.github.com/googleapis/python-bigquery/commit/e37380a959cbd5bb9cbbf6807f0a8ea147e0a713)) +* allow queryJob.result() to be called on a dryRun ([#1015](https://www.github.com/googleapis/python-bigquery/issues/1015)) ([685f06a](https://www.github.com/googleapis/python-bigquery/commit/685f06a5e7b5df17a53e9eb340ff04ecd1e51d1d)) + + +### Documentation + +* document ScriptStatistics and other missing resource classes ([#1023](https://www.github.com/googleapis/python-bigquery/issues/1023)) ([6679109](https://www.github.com/googleapis/python-bigquery/commit/66791093c61f262ea063d2a7950fc643915ee693)) +* fix formatting of generated client docstrings ([#1009](https://www.github.com/googleapis/python-bigquery/issues/1009)) ([f7b0ee4](https://www.github.com/googleapis/python-bigquery/commit/f7b0ee45a664295ccc9f209eeeac122af8de3c80)) + + +### Dependencies + +* allow pyarrow 6.x ([#1031](https://www.github.com/googleapis/python-bigquery/issues/1031)) ([1c2de74](https://www.github.com/googleapis/python-bigquery/commit/1c2de74a55046a343bcf9474f67100a82fb05401)) + ### [2.28.1](https://www.github.com/googleapis/python-bigquery/compare/v2.28.0...v2.28.1) (2021-10-07) diff --git a/docs/conf.py b/docs/conf.py index 3d07b6bf5..512158e19 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -366,8 +366,9 @@ "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), - "pandas": ("http://pandas.pydata.org/pandas-docs/stable/", None), + "dateutil": ("https://dateutil.readthedocs.io/en/latest/", None), "geopandas": ("https://geopandas.org/", None), + "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None), } diff --git a/docs/magics.rst b/docs/magics.rst index bcaad8fa3..aa14c6bfa 100644 --- a/docs/magics.rst +++ b/docs/magics.rst @@ -1,5 +1,34 @@ IPython Magics for BigQuery =========================== +To use these magics, you must first register them. Run the ``%load_ext`` magic +in a Jupyter notebook cell. + +.. code:: + + %load_ext google.cloud.bigquery + +This makes the ``%%bigquery`` magic available. + +Code Samples +------------ + +Running a query: + +.. literalinclude:: ./samples/magics/query.py + :dedent: 4 + :start-after: [START bigquery_jupyter_query] + :end-before: [END bigquery_jupyter_query] + +Running a parameterized query: + +.. literalinclude:: ./samples/magics/query_params_scalars.py + :dedent: 4 + :start-after: [START bigquery_jupyter_query_params_scalars] + :end-before: [END bigquery_jupyter_query_params_scalars] + +API Reference +------------- + .. automodule:: google.cloud.bigquery.magics.magics :members: diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index a30d748bb..1ac04d50c 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -51,6 +51,7 @@ from google.cloud.bigquery.external_config import ExternalSourceFormat from google.cloud.bigquery.format_options import AvroOptions from google.cloud.bigquery.format_options import ParquetOptions +from google.cloud.bigquery.job.base import SessionInfo from google.cloud.bigquery.job import Compression from google.cloud.bigquery.job import CopyJob from google.cloud.bigquery.job import CopyJobConfig @@ -76,6 +77,7 @@ from google.cloud.bigquery.model import ModelReference from google.cloud.bigquery.query import ArrayQueryParameter from google.cloud.bigquery.query import ArrayQueryParameterType +from google.cloud.bigquery.query import ConnectionProperty from google.cloud.bigquery.query import ScalarQueryParameter from google.cloud.bigquery.query import ScalarQueryParameterType from google.cloud.bigquery.query import SqlParameterScalarTypes @@ -108,6 +110,7 @@ "__version__", "Client", # Queries + "ConnectionProperty", "QueryJob", "QueryJobConfig", "ArrayQueryParameter", @@ -137,6 +140,7 @@ "ExtractJobConfig", "LoadJob", "LoadJobConfig", + "SessionInfo", "UnknownJob", # Models "Model", diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index f2a8f34f0..ff586e671 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -19,8 +19,9 @@ import decimal import math import re -from typing import Union +from typing import Optional, Union +from dateutil import relativedelta from google.cloud._helpers import UTC from google.cloud._helpers import _date_from_iso8601_date from google.cloud._helpers import _datetime_from_microseconds @@ -40,6 +41,14 @@ re.VERBOSE, ) +# BigQuery sends INTERVAL data in "canonical format" +# https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types#interval_type +_INTERVAL_PATTERN = re.compile( + r"(?P-?)(?P\d+)-(?P\d+) " + r"(?P-?\d+) " + r"(?P-?)(?P\d+):(?P\d+):(?P\d+)\.?(?P\d*)?$" +) + _BQ_STORAGE_OPTIONAL_READ_SESSION_VERSION = packaging.version.Version("2.6.0") @@ -116,6 +125,41 @@ def _int_from_json(value, field): return int(value) +def _interval_from_json( + value: Optional[str], field +) -> Optional[relativedelta.relativedelta]: + """Coerce 'value' to an interval, if set or not nullable.""" + if not _not_null(value, field): + return None + if value is None: + raise TypeError(f"got {value} for REQUIRED field: {repr(field)}") + + parsed = _INTERVAL_PATTERN.match(value) + if parsed is None: + raise ValueError(f"got interval: '{value}' with unexpected format") + + calendar_sign = -1 if parsed.group("calendar_sign") == "-" else 1 + years = calendar_sign * int(parsed.group("years")) + months = calendar_sign * int(parsed.group("months")) + days = int(parsed.group("days")) + time_sign = -1 if parsed.group("time_sign") == "-" else 1 + hours = time_sign * int(parsed.group("hours")) + minutes = time_sign * int(parsed.group("minutes")) + seconds = time_sign * int(parsed.group("seconds")) + fraction = parsed.group("fraction") + microseconds = time_sign * int(fraction.ljust(6, "0")[:6]) if fraction else 0 + + return relativedelta.relativedelta( + years=years, + months=months, + days=days, + hours=hours, + minutes=minutes, + seconds=seconds, + microseconds=microseconds, + ) + + def _float_from_json(value, field): """Coerce 'value' to a float, if set or not nullable.""" if _not_null(value, field): @@ -252,6 +296,7 @@ def _record_from_json(value, field): _CELLDATA_FROM_JSON = { "INTEGER": _int_from_json, "INT64": _int_from_json, + "INTERVAL": _interval_from_json, "FLOAT": _float_from_json, "FLOAT64": _float_from_json, "NUMERIC": _decimal_from_json, diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 9f3a4f972..3dbfeb264 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -777,13 +777,12 @@ def get_dataset( def get_iam_policy( self, - table: Union[Table, TableReference], + table: Union[Table, TableReference, TableListItem, str], requested_policy_version: int = 1, retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Policy: - if not isinstance(table, (Table, TableReference)): - raise TypeError("table must be a Table or TableReference") + table = _table_arg_to_table_ref(table, default_project=self.project) if requested_policy_version != 1: raise ValueError("only IAM policy version 1 is supported") @@ -806,14 +805,13 @@ def get_iam_policy( def set_iam_policy( self, - table: Union[Table, TableReference], + table: Union[Table, TableReference, TableListItem, str], policy: Policy, updateMask: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Policy: - if not isinstance(table, (Table, TableReference)): - raise TypeError("table must be a Table or TableReference") + table = _table_arg_to_table_ref(table, default_project=self.project) if not isinstance(policy, (Policy)): raise TypeError("policy must be a Policy") @@ -840,13 +838,12 @@ def set_iam_policy( def test_iam_permissions( self, - table: Union[Table, TableReference], + table: Union[Table, TableReference, TableListItem, str], permissions: Sequence[str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Dict[str, Any]: - if not isinstance(table, (Table, TableReference)): - raise TypeError("table must be a Table or TableReference") + table = _table_arg_to_table_ref(table, default_project=self.project) body = {"permissions": permissions} @@ -953,7 +950,7 @@ def get_routine( def get_table( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Table: @@ -963,6 +960,7 @@ def get_table( table (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): A reference to the table to fetch from the BigQuery API. @@ -1728,7 +1726,7 @@ def delete_routine( def delete_table( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, not_found_ok: bool = False, @@ -1742,6 +1740,7 @@ def delete_table( table (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): A reference to the table to delete. If a string is passed in, @@ -2226,7 +2225,7 @@ def api_request(*args, **kwargs): def load_table_from_uri( self, source_uris: Union[str, Sequence[str]], - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], job_id: str = None, job_id_prefix: str = None, location: str = None, @@ -2247,6 +2246,7 @@ def load_table_from_uri( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be loaded. If a string is passed @@ -2308,7 +2308,7 @@ def load_table_from_uri( def load_table_from_file( self, file_obj: BinaryIO, - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], rewind: bool = False, size: int = None, num_retries: int = _DEFAULT_NUM_RETRIES, @@ -2329,6 +2329,7 @@ def load_table_from_file( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be loaded. If a string is passed @@ -2651,7 +2652,7 @@ def load_table_from_dataframe( def load_table_from_json( self, json_rows: Iterable[Dict[str, Any]], - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], num_retries: int = _DEFAULT_NUM_RETRIES, job_id: str = None, job_id_prefix: str = None, @@ -2685,6 +2686,7 @@ def load_table_from_json( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be loaded. If a string is passed @@ -2932,9 +2934,13 @@ def _do_multipart_upload( def copy_table( self, sources: Union[ - Table, TableReference, str, Sequence[Union[Table, TableReference, str]] + Table, + TableReference, + TableListItem, + str, + Sequence[Union[Table, TableReference, TableListItem, str]], ], - destination: Union[Table, TableReference, str], + destination: Union[Table, TableReference, TableListItem, str], job_id: str = None, job_id_prefix: str = None, location: str = None, @@ -2952,11 +2958,13 @@ def copy_table( sources (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ Sequence[ \ Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ] \ ], \ @@ -2965,6 +2973,7 @@ def copy_table( destination (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): Table into which data is to be copied. @@ -3036,7 +3045,7 @@ def copy_table( def extract_table( self, - source: Union[Table, TableReference, Model, ModelReference, str], + source: Union[Table, TableReference, TableListItem, Model, ModelReference, str], destination_uris: Union[str, Sequence[str]], job_id: str = None, job_id_prefix: str = None, @@ -3056,6 +3065,7 @@ def extract_table( source (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ google.cloud.bigquery.model.Model, \ google.cloud.bigquery.model.ModelReference, \ src, \ @@ -3417,7 +3427,7 @@ def insert_rows_from_dataframe( def insert_rows_json( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], json_rows: Sequence[Dict], row_ids: Union[Iterable[str], AutoRowIDs, None] = AutoRowIDs.GENERATE_UUID, skip_invalid_rows: bool = None, @@ -3435,6 +3445,7 @@ def insert_rows_json( table (Union[ \ google.cloud.bigquery.table.Table \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str \ ]): The destination table for the row data, or a reference to it. @@ -3557,7 +3568,7 @@ def insert_rows_json( def list_partitions( self, - table: Union[Table, TableReference, str], + table: Union[Table, TableReference, TableListItem, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = DEFAULT_TIMEOUT, ) -> Sequence[str]: @@ -3567,6 +3578,7 @@ def list_partitions( table (Union[ \ google.cloud.bigquery.table.Table, \ google.cloud.bigquery.table.TableReference, \ + google.cloud.bigquery.table.TableListItem, \ str, \ ]): The table or reference from which to get partition info diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index cecdaa503..8c24f71e7 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -219,6 +219,7 @@ class SqlTypeNames(str, enum.Enum): DATE = "DATE" TIME = "TIME" DATETIME = "DATETIME" + INTERVAL = "INTERVAL" # NOTE: not available in legacy types class WriteDisposition(object): diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 9e381ded6..88d6bec14 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -202,6 +202,19 @@ def script_statistics(self) -> Optional["ScriptStatistics"]: return None return ScriptStatistics(resource) + @property + def session_info(self) -> Optional["SessionInfo"]: + """[Preview] Information of the session if this job is part of one. + + .. versionadded:: 2.29.0 + """ + resource = _helpers._get_sub_prop( + self._properties, ["statistics", "sessionInfo"] + ) + if resource is None: + return None + return SessionInfo(resource) + @property def num_child_jobs(self): """The number of child jobs executed. @@ -990,6 +1003,24 @@ def evaluation_kind(self) -> Optional[str]: return self._properties.get("evaluationKind") +class SessionInfo: + """[Preview] Information of the session if this job is part of one. + + .. versionadded:: 2.29.0 + + Args: + resource (Map[str, Any]): JSON representation of object. + """ + + def __init__(self, resource): + self._properties = resource + + @property + def session_id(self) -> Optional[str]: + """The ID of the session.""" + return self._properties.get("sessionId") + + class UnknownJob(_AsyncJob): """A job whose type cannot be determined.""" diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 6a973bb65..6e6b0cc30 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -18,7 +18,7 @@ import copy import re import typing -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Iterable, List, Optional, Union from google.api_core import exceptions from google.api_core.future import polling as polling_future @@ -31,11 +31,14 @@ from google.cloud.bigquery.enums import KeyResultStatementKind from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery import _helpers -from google.cloud.bigquery.query import _query_param_from_api_repr -from google.cloud.bigquery.query import ArrayQueryParameter -from google.cloud.bigquery.query import ScalarQueryParameter -from google.cloud.bigquery.query import StructQueryParameter -from google.cloud.bigquery.query import UDFResource +from google.cloud.bigquery.query import ( + _query_param_from_api_repr, + ArrayQueryParameter, + ConnectionProperty, + ScalarQueryParameter, + StructQueryParameter, + UDFResource, +) from google.cloud.bigquery.retry import DEFAULT_RETRY, DEFAULT_JOB_RETRY from google.cloud.bigquery.routine import RoutineReference from google.cloud.bigquery.schema import SchemaField @@ -269,6 +272,24 @@ def allow_large_results(self): def allow_large_results(self, value): self._set_sub_prop("allowLargeResults", value) + @property + def connection_properties(self) -> List[ConnectionProperty]: + """Connection properties. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery.FIELDS.connection_properties + + .. versionadded:: 2.29.0 + """ + resource = self._get_sub_prop("connectionProperties", []) + return [ConnectionProperty.from_api_repr(prop) for prop in resource] + + @connection_properties.setter + def connection_properties(self, value: Iterable[ConnectionProperty]): + self._set_sub_prop( + "connectionProperties", [prop.to_api_repr() for prop in value], + ) + @property def create_disposition(self): """google.cloud.bigquery.job.CreateDisposition: Specifies behavior @@ -283,6 +304,27 @@ def create_disposition(self): def create_disposition(self, value): self._set_sub_prop("createDisposition", value) + @property + def create_session(self) -> Optional[bool]: + """[Preview] If :data:`True`, creates a new session, where + :attr:`~google.cloud.bigquery.job.QueryJob.session_info` will contain a + random server generated session id. + + If :data:`False`, runs query with an existing ``session_id`` passed in + :attr:`~google.cloud.bigquery.job.QueryJobConfig.connection_properties`, + otherwise runs query in non-session mode. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationQuery.FIELDS.create_session + + .. versionadded:: 2.29.0 + """ + return self._get_sub_prop("createSession") + + @create_session.setter + def create_session(self, value: Optional[bool]): + self._set_sub_prop("createSession", value) + @property def default_dataset(self): """google.cloud.bigquery.dataset.DatasetReference: the default dataset @@ -613,7 +655,7 @@ def schema_update_options(self, values): @property def script_options(self) -> ScriptOptions: - """Connection properties which can modify the query behavior. + """Options controlling the execution of scripts. https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#scriptoptions """ @@ -694,6 +736,15 @@ def allow_large_results(self): """ return self._configuration.allow_large_results + @property + def connection_properties(self) -> List[ConnectionProperty]: + """See + :attr:`google.cloud.bigquery.job.QueryJobConfig.connection_properties`. + + .. versionadded:: 2.29.0 + """ + return self._configuration.connection_properties + @property def create_disposition(self): """See @@ -701,6 +752,15 @@ def create_disposition(self): """ return self._configuration.create_disposition + @property + def create_session(self) -> Optional[bool]: + """See + :attr:`google.cloud.bigquery.job.QueryJobConfig.create_session`. + + .. versionadded:: 2.29.0 + """ + return self._configuration.create_session + @property def default_dataset(self): """See diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 60670167e..cb2614acc 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -14,15 +14,6 @@ """IPython Magics -To use these magics, you must first register them. Run the ``%load_ext`` magic -in a Jupyter notebook cell. - -.. code:: - - %load_ext google.cloud.bigquery - -This makes the ``%%bigquery`` magic available. - .. function:: %%bigquery IPython cell magic to run a query and display the result as a DataFrame @@ -85,63 +76,6 @@ .. note:: All queries run using this magic will run using the context :attr:`~google.cloud.bigquery.magics.Context.credentials`. - - Examples: - The following examples can be run in an IPython notebook after loading - the bigquery IPython extension (see ``In[1]``) and setting up - Application Default Credentials. - - .. code-block:: none - - In [1]: %load_ext google.cloud.bigquery - - In [2]: %%bigquery - ...: SELECT name, SUM(number) as count - ...: FROM `bigquery-public-data.usa_names.usa_1910_current` - ...: GROUP BY name - ...: ORDER BY count DESC - ...: LIMIT 3 - - Out[2]: name count - ...: ------------------- - ...: 0 James 4987296 - ...: 1 John 4866302 - ...: 2 Robert 4738204 - - In [3]: %%bigquery df --project my-alternate-project --verbose - ...: SELECT name, SUM(number) as count - ...: FROM `bigquery-public-data.usa_names.usa_1910_current` - ...: WHERE gender = 'F' - ...: GROUP BY name - ...: ORDER BY count DESC - ...: LIMIT 3 - Executing query with job ID: bf633912-af2c-4780-b568-5d868058632b - Query executing: 2.61s - Query complete after 2.92s - - In [4]: df - - Out[4]: name count - ...: ---------------------- - ...: 0 Mary 3736239 - ...: 1 Patricia 1568495 - ...: 2 Elizabeth 1519946 - - In [5]: %%bigquery --params {"num": 17} - ...: SELECT @num AS num - - Out[5]: num - ...: ------- - ...: 0 17 - - In [6]: params = {"num": 17} - - In [7]: %%bigquery --params $params - ...: SELECT @num AS num - - Out[7]: num - ...: ------- - ...: 0 17 """ from __future__ import print_function diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index d58d46fd9..2a20d6944 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -18,7 +18,7 @@ import copy import datetime import decimal -from typing import Optional, Union +from typing import Any, Optional, Dict, Union from google.cloud.bigquery.table import _parse_schema_resource from google.cloud.bigquery._helpers import _rows_from_json @@ -31,6 +31,65 @@ ] +class ConnectionProperty: + """A connection-level property to customize query behavior. + + See + https://cloud.google.com/bigquery/docs/reference/rest/v2/ConnectionProperty + + Args: + key: + The key of the property to set, for example, ``'time_zone'`` or + ``'session_id'``. + value: The value of the property to set. + """ + + def __init__(self, key: str = "", value: str = ""): + self._properties = { + "key": key, + "value": value, + } + + @property + def key(self) -> str: + """Name of the property. + + For example: + + * ``time_zone`` + * ``session_id`` + """ + return self._properties["key"] + + @property + def value(self) -> str: + """Value of the property.""" + return self._properties["value"] + + @classmethod + def from_api_repr(cls, resource) -> "ConnectionProperty": + """Construct :class:`~google.cloud.bigquery.query.ConnectionProperty` + from JSON resource. + + Args: + resource: JSON representation. + + Returns: + A connection property. + """ + value = cls() + value._properties = resource + return value + + def to_api_repr(self) -> Dict[str, Any]: + """Construct JSON API representation for the connection property. + + Returns: + JSON mapping + """ + return self._properties + + class UDFResource(object): """Describe a single user-defined function (UDF) resource. diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 91311d332..2bf044010 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -285,7 +285,7 @@ def _key(self): field_type = f"{field_type}({self.precision})" policy_tags = ( - () if self.policy_tags is None else tuple(sorted(self.policy_tags.names)) + None if self.policy_tags is None else tuple(sorted(self.policy_tags.names)) ) return ( @@ -342,7 +342,11 @@ def __hash__(self): return hash(self._key()) def __repr__(self): - return "SchemaField{}".format(self._key()) + key = self._key() + policy_tags = key[-1] + policy_tags_inst = None if policy_tags is None else PolicyTagList(policy_tags) + adjusted_key = key[:-1] + (policy_tags_inst,) + return f"{self.__class__.__name__}{adjusted_key}" def _parse_schema_resource(info): @@ -413,7 +417,7 @@ class PolicyTagList(object): `projects/*/locations/*/taxonomies/*/policyTags/*`. """ - def __init__(self, names=()): + def __init__(self, names: Iterable[str] = ()): self._properties = {} self._properties["names"] = tuple(names) @@ -431,7 +435,7 @@ def _key(self): Returns: Tuple: The contents of this :class:`~google.cloud.bigquery.schema.PolicyTagList`. """ - return tuple(sorted(self._properties.items())) + return tuple(sorted(self._properties.get("names", ()))) def __eq__(self, other): if not isinstance(other, PolicyTagList): @@ -445,7 +449,7 @@ def __hash__(self): return hash(self._key()) def __repr__(self): - return "PolicyTagList{}".format(self._key()) + return f"{self.__class__.__name__}(names={self._key()})" @classmethod def from_api_repr(cls, api_repr: dict) -> "PolicyTagList": @@ -484,5 +488,5 @@ def to_api_repr(self) -> dict: A dictionary representing the PolicyTagList object in serialized form. """ - answer = {"names": [name for name in self.names]} + answer = {"names": list(self.names)} return answer diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 967959b05..877ea53d8 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.28.1" +__version__ = "2.30.1" diff --git a/google/cloud/bigquery_v2/types/model.py b/google/cloud/bigquery_v2/types/model.py index 6e3ca0095..a56b21491 100644 --- a/google/cloud/bigquery_v2/types/model.py +++ b/google/cloud/bigquery_v2/types/model.py @@ -560,14 +560,23 @@ class Cluster(proto.Message): class FeatureValue(proto.Message): r"""Representative value of a single feature within the cluster. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: feature_column (str): The feature column name. numerical_value (google.protobuf.wrappers_pb2.DoubleValue): The numerical feature value. This is the centroid value for this feature. + This field is a member of `oneof`_ ``value``. categorical_value (google.cloud.bigquery_v2.types.Model.ClusteringMetrics.Cluster.FeatureValue.CategoricalValue): The categorical feature value. + This field is a member of `oneof`_ ``value``. """ class CategoricalValue(proto.Message): @@ -784,23 +793,36 @@ class EvaluationMetrics(proto.Message): data was used during training. These are not present for imported models. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: regression_metrics (google.cloud.bigquery_v2.types.Model.RegressionMetrics): Populated for regression models and explicit feedback type matrix factorization models. + This field is a member of `oneof`_ ``metrics``. binary_classification_metrics (google.cloud.bigquery_v2.types.Model.BinaryClassificationMetrics): Populated for binary classification/classifier models. + This field is a member of `oneof`_ ``metrics``. multi_class_classification_metrics (google.cloud.bigquery_v2.types.Model.MultiClassClassificationMetrics): Populated for multi-class classification/classifier models. + This field is a member of `oneof`_ ``metrics``. clustering_metrics (google.cloud.bigquery_v2.types.Model.ClusteringMetrics): Populated for clustering models. + This field is a member of `oneof`_ ``metrics``. ranking_metrics (google.cloud.bigquery_v2.types.Model.RankingMetrics): Populated for implicit feedback type matrix factorization models. + This field is a member of `oneof`_ ``metrics``. arima_forecasting_metrics (google.cloud.bigquery_v2.types.Model.ArimaForecastingMetrics): Populated for ARIMA models. + This field is a member of `oneof`_ ``metrics``. """ regression_metrics = proto.Field( diff --git a/google/cloud/bigquery_v2/types/standard_sql.py b/google/cloud/bigquery_v2/types/standard_sql.py index 69a221c3c..d6c133634 100644 --- a/google/cloud/bigquery_v2/types/standard_sql.py +++ b/google/cloud/bigquery_v2/types/standard_sql.py @@ -35,6 +35,13 @@ class StandardSqlDataType(proto.Message): type={type_kind="STRING"}}, {name="y", type={type_kind="ARRAY", array_element_type="DATE"}} ]}} + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: type_kind (google.cloud.bigquery_v2.types.StandardSqlDataType.TypeKind): Required. The top level type of this field. @@ -42,9 +49,11 @@ class StandardSqlDataType(proto.Message): "INT64", "DATE", "ARRAY"). array_element_type (google.cloud.bigquery_v2.types.StandardSqlDataType): The type of the array's elements, if type_kind = "ARRAY". + This field is a member of `oneof`_ ``sub_type``. struct_type (google.cloud.bigquery_v2.types.StandardSqlStructType): The fields of this struct, in order, if type_kind = "STRUCT". + This field is a member of `oneof`_ ``sub_type``. """ class TypeKind(proto.Enum): diff --git a/noxfile.py b/noxfile.py index d41573407..1879a5cd8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -27,8 +27,8 @@ BLACK_PATHS = ("docs", "google", "samples", "tests", "noxfile.py", "setup.py") DEFAULT_PYTHON_VERSION = "3.8" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] -UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.8", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() # 'docfx' is excluded since it only needs to run in 'docs-presubmit' @@ -69,7 +69,12 @@ def default(session, install_extras=True): constraints_path, ) - install_target = ".[all]" if install_extras else "." + if install_extras and session.python == "3.10": + install_target = ".[bqstorage,pandas,tqdm,opentelemetry]" + elif install_extras: + install_target = ".[all]" + else: + install_target = "." session.install("-e", install_target, "-c", constraints_path) session.install("ipython", "-c", constraints_path) @@ -153,7 +158,11 @@ def system(session): # Data Catalog needed for the column ACL test with a real Policy Tag. session.install("google-cloud-datacatalog", "-c", constraints_path) - session.install("-e", ".[all]", "-c", constraints_path) + if session.python == "3.10": + extras = "[bqstorage,pandas,tqdm,opentelemetry]" + else: + extras = "[all]" + session.install("-e", f".{extras}", "-c", constraints_path) session.install("ipython", "-c", constraints_path) # Run py.test against the system tests. @@ -177,7 +186,11 @@ def snippets(session): session.install("google-cloud-storage", "-c", constraints_path) session.install("grpcio", "-c", constraints_path) - session.install("-e", ".[all]", "-c", constraints_path) + if session.python == "3.10": + extras = "[bqstorage,pandas,tqdm,opentelemetry]" + else: + extras = "[all]" + session.install("-e", f".{extras}", "-c", constraints_path) # Run py.test against the snippets tests. # Skip tests in samples/snippets, as those are run in a different session @@ -186,8 +199,9 @@ def snippets(session): session.run( "py.test", "samples", - "--ignore=samples/snippets", + "--ignore=samples/magics", "--ignore=samples/geography", + "--ignore=samples/snippets", *session.posargs, ) diff --git a/owlbot.py b/owlbot.py index 93620ab98..66eef5b43 100644 --- a/owlbot.py +++ b/owlbot.py @@ -30,8 +30,9 @@ microgenerator=True, split_system_tests=True, intersphinx_dependencies={ - "pandas": "https://pandas.pydata.org/pandas-docs/stable/", + "dateutil": "https://dateutil.readthedocs.io/en/latest/", "geopandas": "https://geopandas.org/", + "pandas": "https://pandas.pydata.org/pandas-docs/stable/", }, ) @@ -47,10 +48,6 @@ # Include custom SNIPPETS_TESTS job for performance. # https://github.com/googleapis/python-bigquery/issues/191 ".kokoro/presubmit/presubmit.cfg", - # Group all renovate PRs together. If this works well, remove this and - # update the shared templates (possibly with configuration option to - # py_library.) - "renovate.json", ], ) diff --git a/renovate.json b/renovate.json index 713c60bb4..c21036d38 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,9 @@ { "extends": [ - "config:base", "group:all", ":preserveSemverRanges" + "config:base", + "group:all", + ":preserveSemverRanges", + ":disableDependencyDashboard" ], "ignorePaths": [".pre-commit-config.yaml"], "pip_requirements": { diff --git a/samples/geography/noxfile_config.py b/samples/geography/noxfile_config.py index 7d2e02346..315bd5be8 100644 --- a/samples/geography/noxfile_config.py +++ b/samples/geography/noxfile_config.py @@ -22,7 +22,12 @@ TEST_CONFIG_OVERRIDE = { # You can opt out from the test for specific Python versions. - "ignored_versions": ["2.7"], + "ignored_versions": [ + "2.7", + # TODO: Enable 3.10 once there is a geopandas/fiona release. + # https://github.com/Toblerity/Fiona/issues/1043 + "3.10", + ], # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index ecd428ab9..e2de86673 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -24,14 +24,12 @@ importlib-metadata==4.8.1 libcst==0.3.21 munch==2.5.0 mypy-extensions==0.4.3 -numpy==1.19.5; python_version < "3.7" -numpy==1.21.2; python_version > "3.6" packaging==21.0 pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' +pandas==1.3.4; python_version >= '3.7' proto-plus==1.19.2 protobuf==3.18.0 -pyarrow==5.0.0 +pyarrow==6.0.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.20 @@ -43,7 +41,7 @@ pytz==2021.1 PyYAML==5.4.1 requests==2.26.0 rsa==4.7.2 -Shapely==1.7.1 +Shapely==1.8.0 six==1.16.0 typing-extensions==3.10.0.2 typing-inspect==0.7.1 diff --git a/samples/magics/__init__.py b/samples/magics/__init__.py new file mode 100644 index 000000000..4fbd93bb2 --- /dev/null +++ b/samples/magics/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/samples/magics/_helpers.py b/samples/magics/_helpers.py new file mode 100644 index 000000000..18a513b99 --- /dev/null +++ b/samples/magics/_helpers.py @@ -0,0 +1,21 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def strip_region_tags(sample_text): + """Remove blank lines and region tags from sample text""" + magic_lines = [ + line for line in sample_text.split("\n") if len(line) > 0 and "# [" not in line + ] + return "\n".join(magic_lines) diff --git a/samples/magics/conftest.py b/samples/magics/conftest.py new file mode 100644 index 000000000..bf8602235 --- /dev/null +++ b/samples/magics/conftest.py @@ -0,0 +1,36 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +interactiveshell = pytest.importorskip("IPython.terminal.interactiveshell") +tools = pytest.importorskip("IPython.testing.tools") + + +@pytest.fixture(scope="session") +def ipython(): + config = tools.default_config() + config.TerminalInteractiveShell.simple_prompt = True + shell = interactiveshell.TerminalInteractiveShell.instance(config=config) + return shell + + +@pytest.fixture(autouse=True) +def ipython_interactive(ipython): + """Activate IPython's builtin hooks + + for the duration of the test scope. + """ + with ipython.builtin_trap: + yield ipython diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py new file mode 100644 index 000000000..93a9122cc --- /dev/null +++ b/samples/magics/noxfile.py @@ -0,0 +1,270 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys +from typing import Callable, Dict, List, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==19.10b0" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +def _determine_local_import_names(start_dir: str) -> List[str]: + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8", "flake8-import-order") + else: + session.install("flake8", "flake8-import-order", "flake8-annotations") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/samples/magics/query.py b/samples/magics/query.py new file mode 100644 index 000000000..c2739eace --- /dev/null +++ b/samples/magics/query.py @@ -0,0 +1,37 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import IPython + +from . import _helpers + + +def query(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + sample = """ + # [START bigquery_jupyter_query] + %%bigquery + SELECT name, SUM(number) as count + FROM `bigquery-public-data.usa_names.usa_1910_current` + GROUP BY name + ORDER BY count DESC + LIMIT 3 + # [END bigquery_jupyter_query] + """ + result = ip.run_cell(_helpers.strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + df = ip.user_ns["_"] # Retrieves last returned object in notebook session + return df diff --git a/samples/magics/query_params_scalars.py b/samples/magics/query_params_scalars.py new file mode 100644 index 000000000..a26f25aea --- /dev/null +++ b/samples/magics/query_params_scalars.py @@ -0,0 +1,38 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import IPython + +from . import _helpers + + +def query_with_parameters(): + ip = IPython.get_ipython() + ip.extension_manager.load_extension("google.cloud.bigquery") + + sample = """ + # [START bigquery_jupyter_query_params_scalars] + %%bigquery --params {"corpus_name": "hamlet", "limit": 10} + SELECT word, SUM(word_count) as count + FROM `bigquery-public-data.samples.shakespeare` + WHERE corpus = @corpus_name + GROUP BY word + ORDER BY count DESC + LIMIT @limit + # [END bigquery_jupyter_query_params_scalars] + """ + result = ip.run_cell(_helpers.strip_region_tags(sample)) + result.raise_error() # Throws an exception if the cell failed. + df = ip.user_ns["_"] # Retrieves last returned object in notebook session + return df diff --git a/samples/magics/query_params_scalars_test.py b/samples/magics/query_params_scalars_test.py new file mode 100644 index 000000000..9b4159667 --- /dev/null +++ b/samples/magics/query_params_scalars_test.py @@ -0,0 +1,23 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas + +from . import query_params_scalars + + +def test_query_with_parameters(): + df = query_params_scalars.query_with_parameters() + assert isinstance(df, pandas.DataFrame) + assert len(df) == 10 diff --git a/samples/magics/query_test.py b/samples/magics/query_test.py new file mode 100644 index 000000000..d20797908 --- /dev/null +++ b/samples/magics/query_test.py @@ -0,0 +1,23 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas + +from . import query + + +def test_query(): + df = query.query() + assert isinstance(df, pandas.DataFrame) + assert len(df) == 3 diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt new file mode 100644 index 000000000..caa48813a --- /dev/null +++ b/samples/magics/requirements-test.txt @@ -0,0 +1,3 @@ +google-cloud-testutils==1.1.0 +pytest==6.2.5 +mock==4.0.3 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt new file mode 100644 index 000000000..5cc7ec33f --- /dev/null +++ b/samples/magics/requirements.txt @@ -0,0 +1,11 @@ +google-cloud-bigquery-storage==2.9.0 +google-auth-oauthlib==0.4.6 +grpcio==1.41.0 +ipython==7.16.1; python_version < '3.7' +ipython==7.29.0; python_version >= '3.7' +matplotlib==3.3.4; python_version < '3.7' +matplotlib==3.5.0rc1; python_version >= '3.7' +pandas==1.1.5; python_version < '3.7' +pandas==1.3.4; python_version >= '3.7' +pyarrow==6.0.0 +pytz==2021.1 diff --git a/samples/query_external_sheets_permanent_table.py b/samples/query_external_sheets_permanent_table.py index 915e9acc3..31143d1b0 100644 --- a/samples/query_external_sheets_permanent_table.py +++ b/samples/query_external_sheets_permanent_table.py @@ -21,6 +21,12 @@ def query_external_sheets_permanent_table(dataset_id): # Create credentials with Drive & BigQuery API scopes. # Both APIs must be enabled for your project before running this code. + # + # If you are using credentials from gcloud, you must authorize the + # application first with the following command: + # + # gcloud auth application-default login \ + # --scopes=https://www.googleapis.com/auth/drive,https://www.googleapis.com/auth/cloud-platform credentials, project = google.auth.default( scopes=[ "https://www.googleapis.com/auth/drive", diff --git a/samples/query_external_sheets_temporary_table.py b/samples/query_external_sheets_temporary_table.py index 1b70e9531..a9d58e388 100644 --- a/samples/query_external_sheets_temporary_table.py +++ b/samples/query_external_sheets_temporary_table.py @@ -22,10 +22,16 @@ def query_external_sheets_temporary_table(): # Create credentials with Drive & BigQuery API scopes. # Both APIs must be enabled for your project before running this code. + # + # If you are using credentials from gcloud, you must authorize the + # application first with the following command: + # + # gcloud auth application-default login \ + # --scopes=https://www.googleapis.com/auth/drive,https://www.googleapis.com/auth/cloud-platform credentials, project = google.auth.default( scopes=[ "https://www.googleapis.com/auth/drive", - "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/cloud-platform", ] ) diff --git a/samples/snippets/create_table_external_hive_partitioned.py b/samples/snippets/create_table_external_hive_partitioned.py new file mode 100644 index 000000000..2ff8a2220 --- /dev/null +++ b/samples/snippets/create_table_external_hive_partitioned.py @@ -0,0 +1,73 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_table_external_hive_partitioned(table_id: str): + original_table_id = table_id + # [START bigquery_create_table_external_hivepartitioned] + # Demonstrates creating an external table with hive partitioning. + + # TODO(developer): Set table_id to the ID of the table to create. + table_id = "your-project.your_dataset.your_table_name" + + # TODO(developer): Set source uri. + # Example file: + # gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/dt=2020-11-15/file1.parquet + uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*" + + # TODO(developer): Set source uri prefix. + source_uri_prefix = ( + "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/" + ) + + # [END bigquery_create_table_external_hivepartitioned] + table_id = original_table_id + # [START bigquery_create_table_external_hivepartitioned] + from google.cloud import bigquery + + # Construct a BigQuery client object. + client = bigquery.Client() + + # Configure the external data source. + external_config = bigquery.ExternalConfig("PARQUET") + external_config.source_uris = [uri] + external_config.autodetect = True + + # Configure partitioning options. + hive_partitioning_opts = bigquery.external_config.HivePartitioningOptions() + + # The layout of the files in here is compatible with the layout requirements for hive partitioning, + # so we can add an optional Hive partitioning configuration to leverage the object paths for deriving + # partitioning column information. + + # For more information on how partitions are extracted, see: + # https://cloud.google.com/bigquery/docs/hive-partitioned-queries-gcs + + # We have a "/dt=YYYY-MM-DD/" path component in our example files as documented above. + # Autolayout will expose this as a column named "dt" of type DATE. + hive_partitioning_opts.mode = "AUTO" + hive_partitioning_opts.require_partition_filter = True + hive_partitioning_opts.source_uri_prefix = source_uri_prefix + + external_config.hive_partitioning = hive_partitioning_opts + + table = bigquery.Table(table_id) + table.external_data_configuration = external_config + + table = client.create_table(table) # Make an API request. + print( + "Created table {}.{}.{}".format(table.project, table.dataset_id, table.table_id) + ) + # [END bigquery_create_table_external_hivepartitioned] + return table diff --git a/samples/snippets/create_table_external_hive_partitioned_test.py b/samples/snippets/create_table_external_hive_partitioned_test.py new file mode 100644 index 000000000..c3cdddb55 --- /dev/null +++ b/samples/snippets/create_table_external_hive_partitioned_test.py @@ -0,0 +1,31 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import create_table_external_hive_partitioned + + +def test_create_table_external_hive_partitioned(capsys, random_table_id): + table = create_table_external_hive_partitioned.create_table_external_hive_partitioned( + random_table_id + ) + + out, _ = capsys.readouterr() + hive_partioning = table.external_data_configuration.hive_partitioning + assert "Created table {}".format(random_table_id) in out + assert ( + hive_partioning.source_uri_prefix + == "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/" + ) + assert hive_partioning.require_partition_filter is True + assert hive_partioning.mode == "AUTO" diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index f9b9d023c..f79552392 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,12 +1,11 @@ -google-cloud-bigquery==2.27.1 google-cloud-bigquery-storage==2.9.0 google-auth-oauthlib==0.4.6 grpcio==1.41.0 ipython==7.16.1; python_version < '3.7' -ipython==7.17.0; python_version >= '3.7' +ipython==7.29.0; python_version >= '3.7' matplotlib==3.3.4; python_version < '3.7' matplotlib==3.4.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.2; python_version >= '3.7' -pyarrow==5.0.0 +pandas==1.3.4; python_version >= '3.7' +pyarrow==6.0.0 pytz==2021.1 diff --git a/setup.py b/setup.py index a7f99b879..a26cf55d5 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ "packaging >= 14.3", "proto-plus >= 1.10.0", # For the legacy proto-based types. "protobuf >= 3.12.0", # For the legacy proto-based types. - "pyarrow >= 3.0.0, < 6.0dev", + "pyarrow >= 3.0.0, < 7.0dev", "requests >= 2.18.0, < 3.0.0dev", ] extras = { @@ -113,6 +113,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Operating System :: OS Independent", "Topic :: Internet", ], @@ -121,7 +122,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6, <3.10", + python_requires=">=3.6, <3.11", include_package_data=True, zip_safe=False, ) diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 6e27172b2..b6b71a8f7 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -18,6 +18,7 @@ pandas==1.0.0 proto-plus==1.10.0 protobuf==3.12.0 pyarrow==3.0.0 +python-dateutil==2.7.2 requests==2.18.0 Shapely==1.6.0 six==1.13.0 diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 4884112ac..8059f21db 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -27,19 +27,6 @@ import uuid from typing import Optional -import psutil -import pytest - -from . import helpers - -try: - import fastavro # to parse BQ storage client results -except ImportError: # pragma: NO COVER - fastavro = None - -import pyarrow -import pyarrow.types - from google.api_core.exceptions import PreconditionFailed from google.api_core.exceptions import BadRequest from google.api_core.exceptions import ClientError @@ -60,12 +47,17 @@ from google.cloud import storage from google.cloud.datacatalog_v1 import types as datacatalog_types from google.cloud.datacatalog_v1 import PolicyTagManagerClient - +import psutil +import pytest +import pyarrow +import pyarrow.types from test_utils.retry import RetryErrors from test_utils.retry import RetryInstanceState from test_utils.retry import RetryResult from test_utils.system import unique_resource_id +from . import helpers + JOB_TIMEOUT = 120 # 2 minutes DATA_PATH = pathlib.Path(__file__).parent.parent / "data" diff --git a/tests/system/test_list_rows.py b/tests/system/test_list_rows.py index 70388059e..4c08958c3 100644 --- a/tests/system/test_list_rows.py +++ b/tests/system/test_list_rows.py @@ -15,6 +15,8 @@ import datetime import decimal +from dateutil import relativedelta + from google.cloud import bigquery from google.cloud.bigquery import enums @@ -64,6 +66,9 @@ def test_list_rows_scalars(bigquery_client: bigquery.Client, scalars_table: str) assert row["datetime_col"] == datetime.datetime(2021, 7, 21, 11, 39, 45) assert row["geography_col"] == "POINT(-122.0838511 37.3860517)" assert row["int64_col"] == 123456789 + assert row["interval_col"] == relativedelta.relativedelta( + years=7, months=11, days=9, hours=4, minutes=15, seconds=37, microseconds=123456 + ) assert row["numeric_col"] == decimal.Decimal("1.23456789") assert row["bignumeric_col"] == decimal.Decimal("10.111213141516171819") assert row["float64_col"] == 1.25 @@ -95,6 +100,9 @@ def test_list_rows_scalars_extreme( assert row["datetime_col"] == datetime.datetime(9999, 12, 31, 23, 59, 59, 999999) assert row["geography_col"] == "POINT(-135 90)" assert row["int64_col"] == 9223372036854775807 + assert row["interval_col"] == relativedelta.relativedelta( + years=-10000, days=-3660000, hours=-87840000 + ) assert row["numeric_col"] == decimal.Decimal(f"9.{'9' * 37}E+28") assert row["bignumeric_col"] == decimal.Decimal(f"9.{'9' * 75}E+37") assert row["float64_col"] == float("Inf") diff --git a/tests/system/test_query.py b/tests/system/test_query.py index 24758595b..649120a7e 100644 --- a/tests/system/test_query.py +++ b/tests/system/test_query.py @@ -27,3 +27,29 @@ def test_dry_run(bigquery_client: bigquery.Client, scalars_table: str): assert query_job.dry_run is True assert query_job.total_bytes_processed > 0 assert len(query_job.schema) > 0 + + +def test_session(bigquery_client: bigquery.Client): + initial_config = bigquery.QueryJobConfig() + initial_config.create_session = True + initial_query = """ + CREATE TEMPORARY TABLE numbers(id INT64) + AS + SELECT * FROM UNNEST([1, 2, 3, 4, 5]) AS id; + """ + initial_job = bigquery_client.query(initial_query, job_config=initial_config) + initial_job.result() + session_id = initial_job.session_info.session_id + assert session_id is not None + + second_config = bigquery.QueryJobConfig() + second_config.connection_properties = [ + bigquery.ConnectionProperty("session_id", session_id), + ] + second_job = bigquery_client.query( + "SELECT COUNT(*) FROM numbers;", job_config=second_config + ) + rows = list(second_job.result()) + + assert len(rows) == 1 + assert rows[0][0] == 5 diff --git a/tests/unit/helpers/test_from_json.py b/tests/unit/helpers/test_from_json.py new file mode 100644 index 000000000..65b054f44 --- /dev/null +++ b/tests/unit/helpers/test_from_json.py @@ -0,0 +1,157 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dateutil.relativedelta import relativedelta +import pytest + +from google.cloud.bigquery.schema import SchemaField + + +def create_field(mode="NULLABLE", type_="IGNORED"): + return SchemaField("test_field", type_, mode=mode) + + +@pytest.fixture +def mut(): + from google.cloud.bigquery import _helpers + + return _helpers + + +def test_interval_from_json_w_none_nullable(mut): + got = mut._interval_from_json(None, create_field()) + assert got is None + + +def test_interval_from_json_w_none_required(mut): + with pytest.raises(TypeError): + mut._interval_from_json(None, create_field(mode="REQUIRED")) + + +def test_interval_from_json_w_invalid_format(mut): + with pytest.raises(ValueError, match="NOT_AN_INTERVAL"): + mut._interval_from_json("NOT_AN_INTERVAL", create_field()) + + +@pytest.mark.parametrize( + ("value", "expected"), + ( + ("0-0 0 0:0:0", relativedelta()), + # SELECT INTERVAL X YEAR + ("-10000-0 0 0:0:0", relativedelta(years=-10000)), + ("-1-0 0 0:0:0", relativedelta(years=-1)), + ("1-0 0 0:0:0", relativedelta(years=1)), + ("10000-0 0 0:0:0", relativedelta(years=10000)), + # SELECT INTERVAL X MONTH + ("-0-11 0 0:0:0", relativedelta(months=-11)), + ("-0-1 0 0:0:0", relativedelta(months=-1)), + ("0-1 0 0:0:0", relativedelta(months=1)), + ("0-11 0 0:0:0", relativedelta(months=11)), + # SELECT INTERVAL X DAY + ("0-0 -3660000 0:0:0", relativedelta(days=-3660000)), + ("0-0 -1 0:0:0", relativedelta(days=-1)), + ("0-0 1 0:0:0", relativedelta(days=1)), + ("0-0 3660000 0:0:0", relativedelta(days=3660000)), + # SELECT INTERVAL X HOUR + ("0-0 0 -87840000:0:0", relativedelta(hours=-87840000)), + ("0-0 0 -1:0:0", relativedelta(hours=-1)), + ("0-0 0 1:0:0", relativedelta(hours=1)), + ("0-0 0 87840000:0:0", relativedelta(hours=87840000)), + # SELECT INTERVAL X MINUTE + ("0-0 0 -0:59:0", relativedelta(minutes=-59)), + ("0-0 0 -0:1:0", relativedelta(minutes=-1)), + ("0-0 0 0:1:0", relativedelta(minutes=1)), + ("0-0 0 0:59:0", relativedelta(minutes=59)), + # SELECT INTERVAL X SECOND + ("0-0 0 -0:0:59", relativedelta(seconds=-59)), + ("0-0 0 -0:0:1", relativedelta(seconds=-1)), + ("0-0 0 0:0:1", relativedelta(seconds=1)), + ("0-0 0 0:0:59", relativedelta(seconds=59)), + # SELECT (INTERVAL -1 SECOND) / 1000000 + ("0-0 0 -0:0:0.000001", relativedelta(microseconds=-1)), + ("0-0 0 -0:0:59.999999", relativedelta(seconds=-59, microseconds=-999999)), + ("0-0 0 -0:0:59.999", relativedelta(seconds=-59, microseconds=-999000)), + ("0-0 0 0:0:59.999", relativedelta(seconds=59, microseconds=999000)), + ("0-0 0 0:0:59.999999", relativedelta(seconds=59, microseconds=999999)), + # Test with multiple digits in each section. + ( + "32-11 45 67:16:23.987654", + relativedelta( + years=32, + months=11, + days=45, + hours=67, + minutes=16, + seconds=23, + microseconds=987654, + ), + ), + ( + "-32-11 -45 -67:16:23.987654", + relativedelta( + years=-32, + months=-11, + days=-45, + hours=-67, + minutes=-16, + seconds=-23, + microseconds=-987654, + ), + ), + # Test with mixed +/- sections. + ( + "9999-9 -999999 9999999:59:59.999999", + relativedelta( + years=9999, + months=9, + days=-999999, + hours=9999999, + minutes=59, + seconds=59, + microseconds=999999, + ), + ), + # Test with fraction that is not microseconds. + ("0-0 0 0:0:42.", relativedelta(seconds=42)), + ("0-0 0 0:0:59.1", relativedelta(seconds=59, microseconds=100000)), + ("0-0 0 0:0:0.12", relativedelta(microseconds=120000)), + ("0-0 0 0:0:0.123", relativedelta(microseconds=123000)), + ("0-0 0 0:0:0.1234", relativedelta(microseconds=123400)), + # Fractional seconds can cause rounding problems if cast to float. See: + # https://github.com/googleapis/python-db-dtypes-pandas/issues/18 + ("0-0 0 0:0:59.876543", relativedelta(seconds=59, microseconds=876543)), + ( + "0-0 0 01:01:01.010101", + relativedelta(hours=1, minutes=1, seconds=1, microseconds=10101), + ), + ( + "0-0 0 09:09:09.090909", + relativedelta(hours=9, minutes=9, seconds=9, microseconds=90909), + ), + ( + "0-0 0 11:11:11.111111", + relativedelta(hours=11, minutes=11, seconds=11, microseconds=111111), + ), + ( + "0-0 0 19:16:23.987654", + relativedelta(hours=19, minutes=16, seconds=23, microseconds=987654), + ), + # Nanoseconds are not expected, but should not cause error. + ("0-0 0 0:0:00.123456789", relativedelta(microseconds=123456)), + ("0-0 0 0:0:59.87654321", relativedelta(seconds=59, microseconds=876543)), + ), +) +def test_w_string_values(mut, value, expected): + got = mut._interval_from_json(value, create_field()) + assert got == expected diff --git a/tests/unit/job/test_base.py b/tests/unit/job/test_base.py index e320c72cb..250be83bb 100644 --- a/tests/unit/job/test_base.py +++ b/tests/unit/job/test_base.py @@ -228,6 +228,15 @@ def test_script_statistics(self): self.assertEqual(stack_frame.end_column, 14) self.assertEqual(stack_frame.text, "QUERY TEXT") + def test_session_info(self): + client = _make_client(project=self.PROJECT) + job = self._make_one(self.JOB_ID, client) + + self.assertIsNone(job.session_info) + job._properties["statistics"] = {"sessionInfo": {"sessionId": "abcdefg"}} + self.assertIsNotNone(job.session_info) + self.assertEqual(job.session_info.session_id, "abcdefg") + def test_transaction_info(self): from google.cloud.bigquery.job.base import TransactionInfo diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index 17baacf5b..4da035b78 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -281,6 +281,8 @@ def test_from_api_repr_bare(self): job = klass.from_api_repr(RESOURCE, client=client) self.assertIs(job._client, client) self._verifyResourceProperties(job, RESOURCE) + self.assertEqual(len(job.connection_properties), 0) + self.assertIsNone(job.create_session) def test_from_api_repr_with_encryption(self): self._setUpConstants() diff --git a/tests/unit/job/test_query_config.py b/tests/unit/job/test_query_config.py index 109cf7e44..7818236f4 100644 --- a/tests/unit/job/test_query_config.py +++ b/tests/unit/job/test_query_config.py @@ -152,6 +152,27 @@ def test_clustering_fields(self): config.clustering_fields = None self.assertIsNone(config.clustering_fields) + def test_connection_properties(self): + from google.cloud.bigquery.job.query import ConnectionProperty + + config = self._get_target_class()() + self.assertEqual(len(config.connection_properties), 0) + + session_id = ConnectionProperty("session_id", "abcd") + time_zone = ConnectionProperty("time_zone", "America/Chicago") + config.connection_properties = [session_id, time_zone] + self.assertEqual(len(config.connection_properties), 2) + self.assertEqual(config.connection_properties[0].key, "session_id") + self.assertEqual(config.connection_properties[0].value, "abcd") + self.assertEqual(config.connection_properties[1].key, "time_zone") + self.assertEqual(config.connection_properties[1].value, "America/Chicago") + + def test_create_session(self): + config = self._get_target_class()() + self.assertIsNone(config.create_session) + config.create_session = True + self.assertTrue(config.create_session) + def test_from_api_repr_empty(self): klass = self._get_target_class() config = klass.from_api_repr({}) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 2504b2838..6dd0a96e9 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1470,7 +1470,7 @@ def test_get_iam_policy_w_invalid_table(self): self.PROJECT, self.DS_ID, self.TABLE_ID, ) - with self.assertRaises(TypeError): + with self.assertRaises(ValueError): client.get_iam_policy(table_resource_string) def test_get_iam_policy_w_invalid_version(self): @@ -1591,7 +1591,7 @@ def test_set_iam_policy_w_invalid_table(self): self.TABLE_ID, ) - with self.assertRaises(TypeError): + with self.assertRaises(ValueError): client.set_iam_policy(table_resource_string, policy) def test_test_iam_permissions(self): @@ -1633,7 +1633,7 @@ def test_test_iam_permissions_w_invalid_table(self): PERMISSIONS = ["bigquery.tables.get", "bigquery.tables.update"] - with self.assertRaises(TypeError): + with self.assertRaises(ValueError): client.test_iam_permissions(table_resource_string, PERMISSIONS) def test_update_dataset_w_invalid_field(self): diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index c845d08c1..e092b90ee 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -526,9 +526,30 @@ def test___hash__not_equals(self): def test___repr__(self): field1 = self._make_one("field1", "STRING") - expected = "SchemaField('field1', 'STRING', 'NULLABLE', None, (), ())" + expected = "SchemaField('field1', 'STRING', 'NULLABLE', None, (), None)" self.assertEqual(repr(field1), expected) + def test___repr__evaluable_no_policy_tags(self): + field = self._make_one("field1", "STRING", "REQUIRED", "Description") + field_repr = repr(field) + SchemaField = self._get_target_class() # needed for eval # noqa + + evaled_field = eval(field_repr) + + assert field == evaled_field + + def test___repr__evaluable_with_policy_tags(self): + policy_tags = PolicyTagList(names=["foo", "bar"]) + field = self._make_one( + "field1", "STRING", "REQUIRED", "Description", policy_tags=policy_tags, + ) + field_repr = repr(field) + SchemaField = self._get_target_class() # needed for eval # noqa + + evaled_field = eval(field_repr) + + assert field == evaled_field + # TODO: dedup with the same class in test_table.py. class _SchemaBase(object): @@ -802,6 +823,34 @@ def test___hash__not_equals(self): set_two = {policy2} self.assertNotEqual(set_one, set_two) + def test___repr__no_tags(self): + policy = self._make_one() + assert repr(policy) == "PolicyTagList(names=())" + + def test___repr__with_tags(self): + policy1 = self._make_one(["foo", "bar", "baz"]) + policy2 = self._make_one(["baz", "bar", "foo"]) + expected_repr = "PolicyTagList(names=('bar', 'baz', 'foo'))" # alphabetical + + assert repr(policy1) == expected_repr + assert repr(policy2) == expected_repr + + def test___repr__evaluable_no_tags(self): + policy = self._make_one(names=[]) + policy_repr = repr(policy) + + evaled_policy = eval(policy_repr) + + assert policy == evaled_policy + + def test___repr__evaluable_with_tags(self): + policy = self._make_one(names=["foo", "bar"]) + policy_repr = repr(policy) + + evaled_policy = eval(policy_repr) + + assert policy == evaled_policy + @pytest.mark.parametrize( "api,expect,key2",