From ddcecfea2cd9133e34af1664d30e058bb09f74e5 Mon Sep 17 00:00:00 2001 From: steffnay Date: Thu, 8 Oct 2020 17:48:23 -0700 Subject: [PATCH 01/12] feat: initial wrapIntegers implementation --- src/bigquery.ts | 177 +++++++++++++++++++++++++++++++++++++++++++++-- src/index.ts | 4 ++ src/job.ts | 19 +++-- src/table.ts | 6 +- test/bigquery.ts | 170 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 365 insertions(+), 11 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 5681ac9c..21b43283 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -95,6 +95,7 @@ export type Query = JobRequest & { maxResults?: number; timeoutMs?: number; pageToken?: string; + wrapIntegers?: boolean | IntegerTypeCastOptions; }; export type QueryOptions = QueryResultsOptions; @@ -171,6 +172,16 @@ export interface BigQueryOptions extends common.GoogleAuthOptions { apiEndpoint?: string; } +export interface IntegerTypeCastOptions { + integerTypeCastFunction?: Function; + fields?: string | string[]; +} + +export type IntegerTypeCastValue = { + integerValue: string | number; + schemaFieldName?: string; +}; + export const PROTOCOL_REGEX = /^(\w*):\/\//; /** @@ -415,6 +426,7 @@ export class BigQuery extends common.Service { static mergeSchemaWithRows_( schema: TableSchema | TableField, rows: TableRow[], + wrapIntegers: boolean | IntegerTypeCastOptions, selectedFields?: string[] ) { if (selectedFields && selectedFields!.length > 0) { @@ -444,10 +456,10 @@ export class BigQuery extends common.Service { let value = field.v; if (schemaField.mode === 'REPEATED') { value = (value as TableRowField[]).map(val => { - return convert(schemaField, val.v, selectedFields); + return convert(schemaField, val.v, wrapIntegers, selectedFields); }); } else { - value = convert(schemaField, value, selectedFields); + value = convert(schemaField, value, wrapIntegers, selectedFields); } // eslint-disable-next-line @typescript-eslint/no-explicit-any const fieldObject: any = {}; @@ -460,6 +472,7 @@ export class BigQuery extends common.Service { schemaField: TableField, // eslint-disable-next-line @typescript-eslint/no-explicit-any value: any, + wrapIntegers: boolean | IntegerTypeCastOptions, selectedFields?: string[] ) { if (is.null(value)) { @@ -483,7 +496,15 @@ export class BigQuery extends common.Service { } case 'INTEGER': case 'INT64': { - value = Number(value); + value = wrapIntegers + ? typeof wrapIntegers === 'object' + ? BigQuery.int( + {integerValue: value, schemaFieldName: schemaField.name}, + wrapIntegers + ) + : BigQuery.int(value) + : Number(value); + break; } case 'NUMERIC': { @@ -494,6 +515,7 @@ export class BigQuery extends common.Service { value = BigQuery.mergeSchemaWithRows_( schemaField, value, + wrapIntegers, selectedFields ).pop(); break; @@ -765,6 +787,29 @@ export class BigQuery extends common.Service { return BigQuery.timestamp(value); } + /** + * A timestamp represents an absolute point in time, independent of any time + * zone or convention such as Daylight Savings Time. + * + * @method BigQuery#timestamp + * @param {Date|string} value The time. + * + * @example + * const {BigQuery} = require('@google-cloud/bigquery'); + * const bigquery = new BigQuery(); + * const timestamp = bigquery.timestamp(new Date()); + */ + static int( + value: IntegerTypeCastValue, + wrapIntegers?: IntegerTypeCastOptions + ) { + return new BigQueryInt(value, wrapIntegers); + } + + int(value: IntegerTypeCastValue, wrapIntegers?: IntegerTypeCastOptions) { + return BigQuery.int(value, wrapIntegers); + } + /** * A geography value represents a surface area on the Earth * in Well-known Text (WKT) format. @@ -797,6 +842,31 @@ export class BigQuery extends common.Service { return BigQuery.geography(value); } + /** + * Convert an INT64 value to Number. + * + * @private + * @param {object} value The INT64 value to convert. + */ + static decodeIntegerValue(value: IntegerTypeCastValue) { + const num = Number(value.integerValue); + if (!Number.isSafeInteger(num)) { + throw new Error( + 'We attempted to return all of the numeric values, but ' + + (value.schemaFieldName ? value.schemaFieldName + ' ' : '') + + 'value ' + + value.integerValue + + " is out of bounds of 'Number.MAX_SAFE_INTEGER'.\n" + + "To prevent this error, please consider passing 'options.wrapNumbers' as\n" + + '{\n' + + ' integerTypeCastFunction: provide \n' + + ' properties: optionally specify property name(s) to be custom casted\n' + + '}\n' + ); + } + return num; + } + /** * Return a value's provided type. * @@ -1826,7 +1896,9 @@ export class BigQuery extends common.Service { query.job.getQueryResults(query, callback as QueryRowsCallback); return; } - this.query(query, {autoPaginate: false}, callback); + const wrapIntegers = query.wrapIntegers ? query.wrapIntegers : false; + delete query.wrapIntegers; + this.query(query, {autoPaginate: false, wrapIntegers}, callback); } } @@ -1847,6 +1919,7 @@ promisifyAll(BigQuery, { 'date', 'datetime', 'geography', + 'int', 'job', 'time', 'timestamp', @@ -1925,3 +1998,99 @@ export class BigQueryTime { this.value = value as string; } } + +/** + * Build a Datastore Int object. For long integers, a string can be provided. + * + * @class + * @param {number|string} value The integer value. + * @param {object} [typeCastOptions] Configuration to convert + * values of `integerValue` type to a custom value. Must provide an + * `integerTypeCastFunction` to handle `integerValue` conversion. + * @param {function} typeCastOptions.integerTypeCastFunction A custom user + * provided function to convert `integerValue`. + * @param {sting|string[]} [typeCastOptions.properties] `Entity` property + * names to be converted using `integerTypeCastFunction`. + * + * @example + * const {Datastore} = require('@google-cloud/datastore'); + * const datastore = new Datastore(); + * const anInt = datastore.int(7); + */ +export class BigQueryInt extends Number { + type: string; + value: string; + typeCastFunction?: Function; + private _schemaFieldName: string | undefined; + constructor( + value: string | number | IntegerTypeCastValue, + typeCastOptions?: IntegerTypeCastOptions + ) { + super(typeof value === 'object' ? value.integerValue : value); + this._schemaFieldName = + typeof value === 'object' ? value.schemaFieldName : undefined; + this.value = + typeof value === 'object' + ? value.integerValue.toString() + : value.toString(); + /** + * @name Int#type + * @type {string} + */ + this.type = 'BigQueryInt'; + /** + * @name Int#value + * @type {string} + */ + + if (typeCastOptions) { + const typeCastFields = typeCastOptions.fields + ? arrify(typeCastOptions.fields) + : undefined; + + const CUSTOM_CAST = + typeCastFields && this._schemaFieldName + ? typeCastFields.includes(this._schemaFieldName) + ? true + : false + : false; + + if (typeof typeCastOptions.integerTypeCastFunction !== 'function') { + throw new Error( + 'integerTypeCastFunction is not a function or was not provided.' + ); + } + + this.typeCastFunction = CUSTOM_CAST + ? typeCastOptions.integerTypeCastFunction + : undefined; + } + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + valueOf(): any { + const shouldCustomCast = this.typeCastFunction ? true : false; + + if (shouldCustomCast) { + try { + return this.typeCastFunction!(this.value); + } catch (error) { + error.message = `integerTypeCastFunction threw an error:\n\n - ${error.message}`; + throw error; + } + } else { + // return this.value; + return BigQuery.decodeIntegerValue({ + integerValue: this.value, + schemaFieldName: this._schemaFieldName, + }); + } + } + + toJSON(): Json { + return {type: this.type, value: this.value}; + } +} + +export interface Json { + [field: string]: string; +} diff --git a/src/index.ts b/src/index.ts index 6237ffad..3a981af3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,6 +19,7 @@ export { BigQueryDateOptions, BigQueryDatetime, BigQueryDatetimeOptions, + BigQueryInt, BigQueryOptions, BigQueryTime, BigQueryTimeOptions, @@ -33,7 +34,10 @@ export { GetJobsCallback, GetJobsOptions, GetJobsResponse, + IntegerTypeCastOptions, + IntegerTypeCastValue, JobRequest, + Json, PagedCallback, PagedRequest, PagedResponse, diff --git a/src/job.ts b/src/job.ts index 9df7c694..8192b17c 100644 --- a/src/job.ts +++ b/src/job.ts @@ -30,6 +30,7 @@ import * as extend from 'extend'; import { BigQuery, + IntegerTypeCastOptions, JobRequest, PagedRequest, QueryRowsCallback, @@ -45,9 +46,10 @@ export type JobOptions = JobRequest; export type CancelCallback = RequestCallback; export type CancelResponse = [bigquery.IJobCancelResponse]; -export type QueryResultsOptions = {job?: Job} & PagedRequest< - bigquery.jobs.IGetQueryResultsParams ->; +export type QueryResultsOptions = { + job?: Job; + wrapIntegers?: boolean | IntegerTypeCastOptions; +} & PagedRequest; /** * @callback QueryResultsCallback @@ -402,9 +404,10 @@ class Job extends Operation { }, options ); - + + const wrapIntegers = qs.wrapIntegers; + delete qs.wrapIntegers; delete qs.job; - this.bigQuery.request( { uri: '/queries/' + this.id, @@ -420,7 +423,11 @@ class Job extends Operation { let rows: any = []; if (resp.schema && resp.rows) { - rows = BigQuery.mergeSchemaWithRows_(resp.schema, resp.rows); + rows = BigQuery.mergeSchemaWithRows_( + resp.schema, + resp.rows, + wrapIntegers! + ); } let nextQuery: {} | null = null; diff --git a/src/table.ts b/src/table.ts index 530a0838..fb791287 100644 --- a/src/table.ts +++ b/src/table.ts @@ -44,6 +44,7 @@ import {GoogleErrorBody} from '@google-cloud/common/build/src/util'; import {Duplex, Writable} from 'stream'; import {JobMetadata} from './job'; import bigquery from './types'; +import {IntegerTypeCastOptions} from './bigquery'; // eslint-disable-next-line @typescript-eslint/no-var-requires const duplexify = require('duplexify'); @@ -102,7 +103,9 @@ export type TableRow = bigquery.ITableRow; export type TableRowField = bigquery.ITableCell; export type TableRowValue = string | TableRow; -export type GetRowsOptions = PagedRequest; +export type GetRowsOptions = PagedRequest & { + wrapIntegers?: boolean | IntegerTypeCastOptions; +}; export type JobLoadMetadata = JobRequest & { format?: string; @@ -1681,6 +1684,7 @@ class Table extends common.ServiceObject { rows = BigQuery.mergeSchemaWithRows_( this.metadata.schema, rows || [], + options.wrapIntegers!, options.selectedFields ? options.selectedFields!.split(',') : [] ); callback!(null, rows, nextQuery, resp); diff --git a/test/bigquery.ts b/test/bigquery.ts index f128056a..d6f0d93e 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -30,8 +30,10 @@ import * as sinon from 'sinon'; import * as uuid from 'uuid'; import { + BigQueryInt, BigQueryDate, Dataset, + IntegerTypeCastValue, Job, PROTOCOL_REGEX, Table, @@ -69,6 +71,7 @@ const fakePfy = Object.assign({}, pfy, { 'date', 'datetime', 'geography', + 'int', 'job', 'time', 'timestamp', @@ -776,6 +779,173 @@ describe('BigQuery', () => { }); }); + describe('int', () => { + const INPUT_STRING = '100'; + + it('should call through to the static method', () => { + const fakeInt = new BigQueryInt('1'); + + sandbox + .stub(BigQuery, 'int') + .withArgs(INPUT_STRING) + .returns(fakeInt); + + const int = bq.int(INPUT_STRING); + assert.strictEqual(int, fakeInt); + }); + + it('should have the correct constructor name', () => { + const int = BigQuery.int(INPUT_STRING); + assert.strictEqual(int.constructor.name, 'BigQueryInt'); + }); + }); + + describe.only('BigQueryInt', () => { + const INPUT_STRING = '9223372036854775807'; + + it('should store the stringified value', () => { + const value = 10; + const int = new BigQueryInt(value); + assert.strictEqual(int.value, value.toString()); + }); + + describe('valueOf', () => { + let valueObject: any; + + beforeEach(() => { + valueObject = { + integerValue: '8', + }; + }); + + describe('integerTypeCastFunction is not provided', () => { + const expectedError = (opts: { + integerValue: string; + schemaFieldName?: string; + }) => { + return new Error( + 'We attempted to return all of the numeric values, but ' + + (opts.schemaFieldName ? opts.schemaFieldName + ' ' : '') + + 'value ' + + opts.integerValue + + " is out of bounds of 'Number.MAX_SAFE_INTEGER'.\n" + + "To prevent this error, please consider passing 'options.wrapNumbers' as\n" + + '{\n' + + ' integerTypeCastFunction: provide \n' + + ' properties: optionally specify property name(s) to be custom casted\n' + + '}\n' + ); + }; + + it('should throw if integerTypeCastOptions is provided but integerTypeCastFunction is not', () => { + assert.throws( + () => new BigQueryInt(valueObject, {}).valueOf(), + /integerTypeCastFunction is not a function or was not provided\./ + ); + }); + + it('should throw if integer value is outside of bounds passing objects', () => { + const largeIntegerValue = (Number.MAX_SAFE_INTEGER + 1).toString(); + const smallIntegerValue = (Number.MIN_SAFE_INTEGER - 1).toString(); + + const valueObject = { + integerValue: largeIntegerValue, + schemaFieldName: 'field', + }; + + const valueObject2 = { + integerValue: smallIntegerValue, + schemaFieldName: 'field', + }; + + assert.throws(() => { + new BigQueryInt(valueObject).valueOf(); + }, expectedError(valueObject)); + + assert.throws(() => { + new BigQueryInt(valueObject2).valueOf(); + }, expectedError(valueObject2)); + }); + + // it('should throw if integer value is outside of bounds passing strings or Numbers', () => { + // const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; + // const smallIntegerValue = Number.MIN_SAFE_INTEGER - 1; + + // // should throw when Number is passed + // assert.throws(() => { + // new entity.Int(largeIntegerValue).valueOf(); + // }, expectedError({integerValue: largeIntegerValue})); + + // // should throw when string is passed + // assert.throws(() => { + // new entity.Int(smallIntegerValue.toString()).valueOf(); + // }, expectedError({integerValue: smallIntegerValue})); + // }); + + // it('should not auto throw on initialization', () => { + // const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; + + // const valueProto = { + // valueType: 'integerValue', + // integerValue: largeIntegerValue, + // }; + + // assert.doesNotThrow(() => { + // const a = new entity.Int(valueProto); + // }, new RegExp(`Integer value ${largeIntegerValue} is out of bounds.`)); + // }); + // }); + + // describe('integerTypeCastFunction is provided', () => { + // it('should throw if integerTypeCastFunction is not a function', () => { + // assert.throws( + // () => + // new entity.Int(valueProto, { + // integerTypeCastFunction: {}, + // }).valueOf(), + // /integerTypeCastFunction is not a function or was not provided\./ + // ); + // }); + + // it('should custom-cast integerValue when integerTypeCastFunction is provided', () => { + // const stub = sinon.stub(); + + // new entity.Int(valueProto, { + // integerTypeCastFunction: stub, + // }).valueOf(); + // assert.ok(stub.calledOnce); + // }); + + // it('should custom-cast integerValue if `properties` specified by user', () => { + // const stub = sinon.stub(); + // Object.assign(valueProto, { + // propertyName: 'thisValue', + // }); + + // new entity.Int(valueProto, { + // integerTypeCastFunction: stub, + // properties: 'thisValue', + // }).valueOf(); + // assert.ok(stub.calledOnce); + // }); + + // it('should not custom-cast integerValue if `properties` not specified by user', () => { + // const stub = sinon.stub(); + + // Object.assign(valueProto, { + // propertyName: 'thisValue', + // }); + + // new entity.Int(valueProto, { + // integerTypeCastFunction: stub, + // properties: 'thatValue', + // }).valueOf(); + // assert.ok(stub.notCalled); + // }); + }); + }); + }); + describe('getTypeDescriptorFromValue_', () => { it('should return correct types', () => { assert.strictEqual( From 6565f2222c55e157935dbeb7cf81596126c8f00a Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 12 Oct 2020 01:31:33 -0700 Subject: [PATCH 02/12] update logic --- src/bigquery.ts | 24 ++++++++++++------------ src/job.ts | 6 ++++-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 21b43283..a9d6936d 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -504,7 +504,6 @@ export class BigQuery extends common.Service { ) : BigQuery.int(value) : Number(value); - break; } case 'NUMERIC': { @@ -2044,24 +2043,25 @@ export class BigQueryInt extends Number { */ if (typeCastOptions) { + if (typeof typeCastOptions.integerTypeCastFunction !== 'function') { + throw new Error( + 'integerTypeCastFunction is not a function or was not provided.' + ); + } + const typeCastFields = typeCastOptions.fields ? arrify(typeCastOptions.fields) : undefined; - const CUSTOM_CAST = - typeCastFields && this._schemaFieldName - ? typeCastFields.includes(this._schemaFieldName) - ? true - : false - : false; + let customCast = true; - if (typeof typeCastOptions.integerTypeCastFunction !== 'function') { - throw new Error( - 'integerTypeCastFunction is not a function or was not provided.' - ); + if (typeCastFields) { + customCast = this._schemaFieldName ? typeCastFields.includes(this._schemaFieldName) + ? true : false + : false } - this.typeCastFunction = CUSTOM_CAST + this.typeCastFunction = customCast ? typeCastOptions.integerTypeCastFunction : undefined; } diff --git a/src/job.ts b/src/job.ts index 8192b17c..d578af10 100644 --- a/src/job.ts +++ b/src/job.ts @@ -405,9 +405,11 @@ class Job extends Operation { options ); - const wrapIntegers = qs.wrapIntegers; + const wrapIntegers = qs.wrapIntegers ? qs.wrapIntegers : false; delete qs.wrapIntegers; + delete qs.job; + this.bigQuery.request( { uri: '/queries/' + this.id, @@ -426,7 +428,7 @@ class Job extends Operation { rows = BigQuery.mergeSchemaWithRows_( resp.schema, resp.rows, - wrapIntegers! + wrapIntegers ); } From d7a920c29ae400a5774f88bf906e8f85bbb147ea Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 12 Oct 2020 05:53:29 -0700 Subject: [PATCH 03/12] Adds tests --- src/bigquery.ts | 12 ++- src/job.ts | 2 +- src/table.ts | 5 +- test/bigquery.ts | 250 +++++++++++++++++++++++++++++++---------------- test/job.ts | 37 ++++++- test/table.ts | 37 ++++++- 6 files changed, 244 insertions(+), 99 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index a9d6936d..59c725b3 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -2056,14 +2056,16 @@ export class BigQueryInt extends Number { let customCast = true; if (typeCastFields) { - customCast = this._schemaFieldName ? typeCastFields.includes(this._schemaFieldName) - ? true : false - : false + customCast = this._schemaFieldName + ? typeCastFields.includes(this._schemaFieldName) + ? true + : false + : false; } this.typeCastFunction = customCast - ? typeCastOptions.integerTypeCastFunction - : undefined; + ? typeCastOptions.integerTypeCastFunction + : undefined; } } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/src/job.ts b/src/job.ts index d578af10..7a03a177 100644 --- a/src/job.ts +++ b/src/job.ts @@ -404,7 +404,7 @@ class Job extends Operation { }, options ); - + const wrapIntegers = qs.wrapIntegers ? qs.wrapIntegers : false; delete qs.wrapIntegers; diff --git a/src/table.ts b/src/table.ts index fb791287..33c5ded8 100644 --- a/src/table.ts +++ b/src/table.ts @@ -1671,6 +1671,8 @@ class Table extends common.ServiceObject { typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : cb; + const wrapIntegers = options.wrapIntegers ? options.wrapIntegers : false; + delete options.wrapIntegers; const onComplete = ( err: Error | null, rows: TableRow[] | null, @@ -1681,10 +1683,11 @@ class Table extends common.ServiceObject { callback!(err, null, null, resp); return; } + rows = BigQuery.mergeSchemaWithRows_( this.metadata.schema, rows || [], - options.wrapIntegers!, + wrapIntegers, options.selectedFields ? options.selectedFields!.split(',') : [] ); callback!(null, rows, nextQuery, resp); diff --git a/test/bigquery.ts b/test/bigquery.ts index d6f0d93e..b8946003 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -32,8 +32,8 @@ import * as uuid from 'uuid'; import { BigQueryInt, BigQueryDate, - Dataset, IntegerTypeCastValue, + Dataset, Job, PROTOCOL_REGEX, Table, @@ -575,12 +575,55 @@ describe('BigQuery', () => { }); const rawRows = rows.map(x => x.raw); - const mergedRows = BigQuery.mergeSchemaWithRows_(schemaObject, rawRows); + const mergedRows = BigQuery.mergeSchemaWithRows_( + schemaObject, + rawRows, + false + ); mergedRows.forEach((mergedRow: {}, index: number) => { assert.deepStrictEqual(mergedRow, rows[index].expected); }); }); + + it('should wrap integers with option', () => { + const wrapIntegersBoolean = true; + const wrapIntegersObject = {}; + const fakeInt = new BigQueryInt(100); + + const SCHEMA_OBJECT = { + fields: [{name: 'fave_number', type: 'INTEGER'}], + } as {fields: TableField[]}; + + const rows = { + raw: { + f: [{v: 100}], + }, + expected: { + fave_number: fakeInt, + }, + }; + + sandbox.stub(BigQuery, 'int').returns(fakeInt); + + let mergedRows = BigQuery.mergeSchemaWithRows_( + SCHEMA_OBJECT, + rows.raw, + wrapIntegersBoolean + ); + mergedRows.forEach((mergedRow: {}) => { + assert.deepStrictEqual(mergedRow, rows.expected); + }); + + mergedRows = BigQuery.mergeSchemaWithRows_( + SCHEMA_OBJECT, + rows.raw, + wrapIntegersObject + ); + mergedRows.forEach((mergedRow: {}) => { + assert.deepStrictEqual(mergedRow, rows.expected); + }); + }); }); describe('date', () => { @@ -783,7 +826,7 @@ describe('BigQuery', () => { const INPUT_STRING = '100'; it('should call through to the static method', () => { - const fakeInt = new BigQueryInt('1'); + const fakeInt = new BigQueryInt('100'); sandbox .stub(BigQuery, 'int') @@ -800,9 +843,7 @@ describe('BigQuery', () => { }); }); - describe.only('BigQueryInt', () => { - const INPUT_STRING = '9223372036854775807'; - + describe('BigQueryInt', () => { it('should store the stringified value', () => { const value = 10; const int = new BigQueryInt(value); @@ -810,17 +851,17 @@ describe('BigQuery', () => { }); describe('valueOf', () => { - let valueObject: any; + let valueObject: IntegerTypeCastValue; beforeEach(() => { valueObject = { - integerValue: '8', + integerValue: 8, }; }); describe('integerTypeCastFunction is not provided', () => { const expectedError = (opts: { - integerValue: string; + integerValue: string | number; schemaFieldName?: string; }) => { return new Error( @@ -867,81 +908,101 @@ describe('BigQuery', () => { }, expectedError(valueObject2)); }); - // it('should throw if integer value is outside of bounds passing strings or Numbers', () => { - // const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; - // const smallIntegerValue = Number.MIN_SAFE_INTEGER - 1; - - // // should throw when Number is passed - // assert.throws(() => { - // new entity.Int(largeIntegerValue).valueOf(); - // }, expectedError({integerValue: largeIntegerValue})); - - // // should throw when string is passed - // assert.throws(() => { - // new entity.Int(smallIntegerValue.toString()).valueOf(); - // }, expectedError({integerValue: smallIntegerValue})); - // }); - - // it('should not auto throw on initialization', () => { - // const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; - - // const valueProto = { - // valueType: 'integerValue', - // integerValue: largeIntegerValue, - // }; - - // assert.doesNotThrow(() => { - // const a = new entity.Int(valueProto); - // }, new RegExp(`Integer value ${largeIntegerValue} is out of bounds.`)); - // }); - // }); - - // describe('integerTypeCastFunction is provided', () => { - // it('should throw if integerTypeCastFunction is not a function', () => { - // assert.throws( - // () => - // new entity.Int(valueProto, { - // integerTypeCastFunction: {}, - // }).valueOf(), - // /integerTypeCastFunction is not a function or was not provided\./ - // ); - // }); - - // it('should custom-cast integerValue when integerTypeCastFunction is provided', () => { - // const stub = sinon.stub(); - - // new entity.Int(valueProto, { - // integerTypeCastFunction: stub, - // }).valueOf(); - // assert.ok(stub.calledOnce); - // }); - - // it('should custom-cast integerValue if `properties` specified by user', () => { - // const stub = sinon.stub(); - // Object.assign(valueProto, { - // propertyName: 'thisValue', - // }); - - // new entity.Int(valueProto, { - // integerTypeCastFunction: stub, - // properties: 'thisValue', - // }).valueOf(); - // assert.ok(stub.calledOnce); - // }); - - // it('should not custom-cast integerValue if `properties` not specified by user', () => { - // const stub = sinon.stub(); - - // Object.assign(valueProto, { - // propertyName: 'thisValue', - // }); - - // new entity.Int(valueProto, { - // integerTypeCastFunction: stub, - // properties: 'thatValue', - // }).valueOf(); - // assert.ok(stub.notCalled); - // }); + it('should throw if integer value is outside of bounds passing strings or Numbers', () => { + const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; + const smallIntegerValue = Number.MIN_SAFE_INTEGER - 1; + + // should throw when Number is passed + assert.throws(() => { + new BigQueryInt(largeIntegerValue).valueOf(); + }, expectedError({integerValue: largeIntegerValue})); + + // should throw when string is passed + assert.throws(() => { + new BigQueryInt(smallIntegerValue.toString()).valueOf(); + }, expectedError({integerValue: smallIntegerValue})); + }); + + it('should not auto throw on initialization', () => { + const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; + + const valueObject = { + integerValue: largeIntegerValue, + }; + + assert.doesNotThrow(() => { + new BigQueryInt(valueObject); + }, new RegExp(`Integer value ${largeIntegerValue} is out of bounds.`)); + }); + + describe('integerTypeCastFunction is provided', () => { + it('should throw if integerTypeCastFunction is not a function', () => { + assert.throws( + () => + new BigQueryInt(valueObject, { + integerTypeCastFunction: {} as Function, + }).valueOf(), + /integerTypeCastFunction is not a function or was not provided\./ + ); + }); + + it('should custom-cast integerValue when integerTypeCastFunction is provided', () => { + const stub = sinon.stub(); + + new BigQueryInt(valueObject, { + integerTypeCastFunction: stub, + }).valueOf(); + assert.ok(stub.calledOnce); + }); + + it('should custom-cast integerValue if in `fields` specified by user', () => { + const stub = sinon.stub(); + + Object.assign(valueObject, { + schemaFieldName: 'funField', + }); + + new BigQueryInt(valueObject, { + integerTypeCastFunction: stub, + fields: 'funField', + }).valueOf(); + assert.ok(stub.calledOnce); + }); + + it('should not custom-cast integerValue if not in `fields` specified by user', () => { + const stub = sinon.stub(); + + Object.assign(valueObject, { + schemaFieldName: 'funField', + }); + + new BigQueryInt(valueObject, { + integerTypeCastFunction: stub, + fields: 'unFunField', + }).valueOf(); + assert.ok(stub.notCalled); + }); + + it('should catch integerTypeCastFunction error and throw', () => { + const error = new Error('My bad!'); + const stub = sinon.stub().throws(error); + assert.throws( + () => + new BigQueryInt(valueObject, { + integerTypeCastFunction: stub, + }).valueOf(), + /integerTypeCastFunction threw an error:/ + ); + }); + }); + }); + + describe('toJSON', () => { + it('should return correct JSON', () => { + const expected = {type: 'BigQueryInt', value: '8'}; + const JSON = new BigQueryInt(valueObject).toJSON(); + assert.deepStrictEqual(JSON, expected); + }); }); }); }); @@ -2480,7 +2541,7 @@ describe('BigQuery', () => { assert( queryStub.calledOnceWithExactly( query, - {autoPaginate: false}, + {autoPaginate: false, wrapIntegers: false}, sinon.match.func ) ); @@ -2497,6 +2558,25 @@ describe('BigQuery', () => { assert(cbStub.calledOnceWithExactly(query, sinon.match.func)); assert(queryStub.notCalled); }); + + it('should pass wrapIntegers if supplied', done => { + const statement = 'SELECT'; + const wrapIntegers = { + integerValue: 100, + }; + const query = { + query: statement, + wrapIntegers, + }; + bq.queryAsStream_(query, done); + assert( + queryStub.calledOnceWithExactly( + query, + {autoPaginate: false, wrapIntegers}, + sinon.match.func + ) + ); + }); }); describe('#sanitizeEndpoint', () => { diff --git a/test/job.ts b/test/job.ts index dee9c2b8..5e8989e8 100644 --- a/test/job.ts +++ b/test/job.ts @@ -196,7 +196,11 @@ describe('BigQuery/Job', () => { callback(null, RESPONSE); }; - BIGQUERY.mergeSchemaWithRows_ = (schema: {}, rows: {}) => { + BIGQUERY.mergeSchemaWithRows_ = ( + schema: {}, + rows: {}, + wrapIntegers: {} + ) => { return rows; }; }); @@ -291,9 +295,10 @@ describe('BigQuery/Job', () => { sandbox .stub(BigQuery, 'mergeSchemaWithRows_') - .callsFake((schema, rows) => { + .callsFake((schema, rows, wrapIntegers) => { assert.strictEqual(schema, response.schema); assert.strictEqual(rows, response.rows); + assert.strictEqual(wrapIntegers, false); return mergedRows; }); @@ -304,6 +309,34 @@ describe('BigQuery/Job', () => { }); }); + it('it should wrap integers', done => { + const response = { + schema: {}, + rows: [], + }; + + const mergedRows: Array<{}> = []; + + const options = {wrapIntegers: true}; + const expectedOptions = Object.assign({location: undefined}); + + BIGQUERY.request = (reqOpts: DecorateRequestOptions) => { + assert.deepStrictEqual(reqOpts.qs, expectedOptions); + done(); + }; + + sandbox + .stub(BigQuery, 'mergeSchemaWithRows_') + .callsFake((schema, rows, wrapIntegers) => { + assert.strictEqual(schema, response.schema); + assert.strictEqual(rows, response.rows); + assert.strictEqual(wrapIntegers, true); + return mergedRows; + }); + + job.getQueryResults(options, assert.ifError); + }); + it('should return the query when the job is not complete', done => { BIGQUERY.request = ( reqOpts: DecorateRequestOptions, diff --git a/test/table.ts b/test/table.ts index 2434fe58..56a9fde7 100644 --- a/test/table.ts +++ b/test/table.ts @@ -149,9 +149,11 @@ describe('BigQuery/Table', () => { table = new Table(DATASET, TABLE_ID); table.bigQuery.request = util.noop; table.bigQuery.createJob = util.noop; - sandbox.stub(BigQuery, 'mergeSchemaWithRows_').callsFake((schema, rows) => { - return rows; - }); + sandbox + .stub(BigQuery, 'mergeSchemaWithRows_') + .callsFake((schema, rows, wrapIntegers) => { + return rows; + }); }); afterEach(() => sandbox.restore()); @@ -1894,6 +1896,7 @@ describe('BigQuery/Table', () => { // Using "Stephen" so you know who to blame for these tests. const rows = [{f: [{v: 'stephen'}]}]; const schema = {fields: [{name: 'name', type: 'string'}]}; + const wrapIntegers = false; const mergedRows = [{name: 'stephen'}]; beforeEach(() => { @@ -1907,9 +1910,10 @@ describe('BigQuery/Table', () => { sandbox.restore(); sandbox .stub(BigQuery, 'mergeSchemaWithRows_') - .callsFake((schema_, rows_) => { + .callsFake((schema_, rows_, wrapIntegers_) => { assert.strictEqual(schema_, schema); assert.strictEqual(rows_, rows); + assert.strictEqual(wrapIntegers_, wrapIntegers); return mergedRows; }); }); @@ -1963,6 +1967,7 @@ describe('BigQuery/Table', () => { it('should return schema-merged rows', done => { const rows = [{f: [{v: 'stephen'}]}]; const schema = {fields: [{name: 'name', type: 'string'}]}; + const wrapIntegers = false; const merged = [{name: 'stephen'}]; table.metadata = {schema}; @@ -1974,9 +1979,10 @@ describe('BigQuery/Table', () => { sandbox.restore(); sandbox .stub(BigQuery, 'mergeSchemaWithRows_') - .callsFake((schema_, rows_) => { + .callsFake((schema_, rows_, wrapIntegers_) => { assert.strictEqual(schema_, schema); assert.strictEqual(rows_, rows); + assert.strictEqual(wrapIntegers_, wrapIntegers); return merged; }); @@ -2132,6 +2138,27 @@ describe('BigQuery/Table', () => { done(); }); }); + + it('should wrap integers', done => { + const wrapIntegers = {integerTypeCastFunction: sinon.stub()}; + const options = {wrapIntegers}; + const merged = [{name: 'stephen'}]; + + table.request = (reqOpts: DecorateRequestOptions, callback: Function) => { + assert.deepStrictEqual(reqOpts.qs, {}); + callback(null, {}); + }; + + sandbox.restore(); + sandbox + .stub(BigQuery, 'mergeSchemaWithRows_') + .callsFake((schema_, rows_, wrapIntegers_) => { + assert.strictEqual(wrapIntegers_, wrapIntegers); + return merged; + }); + + table.getRows(options, done); + }); }); describe('insert', () => { From 4f62dd0d78240950625647e63ce00ef1c12a7ab4 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 12 Oct 2020 07:59:04 -0700 Subject: [PATCH 04/12] update tests & doc strings --- src/bigquery.ts | 91 ++++++++++++++++++++++++++++++------------------ src/job.ts | 7 +++- src/table.ts | 5 +++ test/bigquery.ts | 24 ++++++++----- 4 files changed, 84 insertions(+), 43 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 59c725b3..cecfe85a 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -173,7 +173,7 @@ export interface BigQueryOptions extends common.GoogleAuthOptions { } export interface IntegerTypeCastOptions { - integerTypeCastFunction?: Function; + integerTypeCastFunction: Function; fields?: string | string[]; } @@ -419,6 +419,12 @@ export class BigQuery extends common.Service { * * @param {object} schema * @param {array} rows + * @param {boolean|IntegerTypeCastOptions} wrapIntegers Wrap values of + * 'INT64' type in {@link BigQueryInt} objects. + * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. + * If an `object`, this will return a value returned by + * `wrapNumbers.integerTypeCastFunction`. + * Please see {@link IntegerTypeCastOptions} for options descriptions. * @param {array} selectedFields List of fields to return. * If unspecified, all fields are returned. * @returns {array} Fields using their matching names from the table's schema. @@ -501,7 +507,7 @@ export class BigQuery extends common.Service { ? BigQuery.int( {integerValue: value, schemaFieldName: schemaField.name}, wrapIntegers - ) + ).valueOf() : BigQuery.int(value) : Number(value); break; @@ -787,26 +793,43 @@ export class BigQuery extends common.Service { } /** - * A timestamp represents an absolute point in time, independent of any time - * zone or convention such as Daylight Savings Time. + * A BigQueryInt wraps 'INT64' values. Can be used to maintain precision. * - * @method BigQuery#timestamp - * @param {Date|string} value The time. + * @method BigQuery#int + * @param {string|number|IntegerTypeCastValue} value The INT64 value to convert. + * @param {IntegerTypeCastOptions} typeCastOptions Configuration to convert + * value. Must provide an `integerTypeCastFunction` to handle conversion. * * @example * const {BigQuery} = require('@google-cloud/bigquery'); * const bigquery = new BigQuery(); - * const timestamp = bigquery.timestamp(new Date()); + * + * const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; + * + * customTypeCast = (value) => { + * return value.split(''); + * }; + * + * const wrapIntegers = { + * integerTypeCastFunction: customTypeCast + * }; + * + * const bigQueryInt = bigquery.int(largeIntegerValue, wrapIntegers); + * + * const wrappedValue = bigQueryInt.valueOf(); */ static int( - value: IntegerTypeCastValue, - wrapIntegers?: IntegerTypeCastOptions + value: string | number | IntegerTypeCastValue, + typeCastOptions?: IntegerTypeCastOptions ) { - return new BigQueryInt(value, wrapIntegers); + return new BigQueryInt(value, typeCastOptions); } - int(value: IntegerTypeCastValue, wrapIntegers?: IntegerTypeCastOptions) { - return BigQuery.int(value, wrapIntegers); + int( + value: string | number | IntegerTypeCastValue, + typeCastOptions?: IntegerTypeCastOptions + ) { + return BigQuery.int(value, typeCastOptions); } /** @@ -847,7 +870,7 @@ export class BigQuery extends common.Service { * @private * @param {object} value The INT64 value to convert. */ - static decodeIntegerValue(value: IntegerTypeCastValue) { + static decodeIntegerValue_(value: IntegerTypeCastValue) { const num = Number(value.integerValue); if (!Number.isSafeInteger(num)) { throw new Error( @@ -1010,6 +1033,8 @@ export class BigQuery extends common.Service { * @see [Jobs.query API Reference Docs (see `queryParameters`)]{@link https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#request-body} * * @param {*} value The value. + * @param {string | ProvidedTypeStruct | ProvidedTypeArray} providedType Provided + * query parameter type. * @returns {object} A properly-formed `queryParameter` object. */ static valueToQueryParameter_( @@ -1779,6 +1804,12 @@ export class BigQuery extends common.Service { * complete, in milliseconds, before returning. Default is to return * immediately. If the timeout passes before the job completes, the * request will fail with a `TIMEOUT` error. + * @param {boolean|IntegerTypeCastOptions} [options.wrapIntegers=false] Wrap values + * of 'INT64' type in {@link BigQueryInt} objects. + * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. + * If an `object`, this will return a value returned by + * `wrapNumbers.integerTypeCastFunction`. + * Please see {@link IntegerTypeCastOptions} for options descriptions. * @param {function} [callback] The callback function. * @param {?error} callback.err An error returned while making this request * @param {array} callback.rows The list of results from your query. @@ -1999,22 +2030,22 @@ export class BigQueryTime { } /** - * Build a Datastore Int object. For long integers, a string can be provided. + * Build a BigQueryInt object. For long integers, a string can be provided. * * @class - * @param {number|string} value The integer value. + * @param {string|number|IntegerTypeCastValue} value The integer value. * @param {object} [typeCastOptions] Configuration to convert - * values of `integerValue` type to a custom value. Must provide an - * `integerTypeCastFunction` to handle `integerValue` conversion. + * values of 'INT64' type to a custom value. Must provide an + * `integerTypeCastFunction` to handle conversion. * @param {function} typeCastOptions.integerTypeCastFunction A custom user - * provided function to convert `integerValue`. - * @param {sting|string[]} [typeCastOptions.properties] `Entity` property + * provided function to convert value. + * @param {string|string[]} [typeCastOptions.fields] Schema field * names to be converted using `integerTypeCastFunction`. * * @example - * const {Datastore} = require('@google-cloud/datastore'); - * const datastore = new Datastore(); - * const anInt = datastore.int(7); + * const {BigQuery} = require('@google-cloud/bigquery'); + * const bigquery = new BigQuery(); + * const anInt = bigquery.int(7); */ export class BigQueryInt extends Number { type: string; @@ -2032,15 +2063,8 @@ export class BigQueryInt extends Number { typeof value === 'object' ? value.integerValue.toString() : value.toString(); - /** - * @name Int#type - * @type {string} - */ + this.type = 'BigQueryInt'; - /** - * @name Int#value - * @type {string} - */ if (typeCastOptions) { if (typeof typeCastOptions.integerTypeCastFunction !== 'function') { @@ -2063,9 +2087,8 @@ export class BigQueryInt extends Number { : false; } - this.typeCastFunction = customCast - ? typeCastOptions.integerTypeCastFunction - : undefined; + customCast && + (this.typeCastFunction = typeCastOptions.integerTypeCastFunction); } } // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -2081,7 +2104,7 @@ export class BigQueryInt extends Number { } } else { // return this.value; - return BigQuery.decodeIntegerValue({ + return BigQuery.decodeIntegerValue_({ integerValue: this.value, schemaFieldName: this._schemaFieldName, }); diff --git a/src/job.ts b/src/job.ts index 7a03a177..feb877ce 100644 --- a/src/job.ts +++ b/src/job.ts @@ -339,7 +339,12 @@ class Job extends Operation { * @param {number} [options.timeoutMs] How long to wait for the query to * complete, in milliseconds, before returning. Default is to return * immediately. If the timeout passes before the job completes, the - * request will fail with a `TIMEOUT` error. + * request will fail with a `TIMEOUT` error. + * @param {boolean|IntegerTypeCastOptions} [options.wrapIntegers=false] Wrap values + * of 'INT64' type in {@link BigQueryInt} objects. + * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. + * If an `object`, this will return a value returned by + * `wrapNumbers.integerTypeCastFunction`. * @param {QueryResultsCallback|ManualQueryResultsCallback} [callback] The * callback function. If `autoPaginate` is set to false a * {@link ManualQueryResultsCallback} should be used. diff --git a/src/table.ts b/src/table.ts index 33c5ded8..049b450e 100644 --- a/src/table.ts +++ b/src/table.ts @@ -1624,6 +1624,11 @@ class Table extends common.ServiceObject { * automatically. * @param {number} [options.maxApiCalls] Maximum number of API calls to make. * @param {number} [options.maxResults] Maximum number of results to return. + * @param {boolean|IntegerTypeCastOptions} [options.wrapIntegers=false] Wrap values + * of 'INT64' type in {@link BigQueryInt} objects. + * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. + * If an `object`, this will return a value returned by + * `wrapNumbers.integerTypeCastFunction`. * @param {function} [callback] The callback function. * @param {?error} callback.err An error returned while making this request * @param {array} callback.rows The table data from specified set of rows. diff --git a/test/bigquery.ts b/test/bigquery.ts index b8946003..9b8125a7 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -33,6 +33,7 @@ import { BigQueryInt, BigQueryDate, IntegerTypeCastValue, + IntegerTypeCastOptions, Dataset, Job, PROTOCOL_REGEX, @@ -599,9 +600,12 @@ describe('BigQuery', () => { raw: { f: [{v: 100}], }, - expected: { + expectedBool: { fave_number: fakeInt, }, + expectedObj: { + fave_number: fakeInt.valueOf(), + }, }; sandbox.stub(BigQuery, 'int').returns(fakeInt); @@ -612,7 +616,7 @@ describe('BigQuery', () => { wrapIntegersBoolean ); mergedRows.forEach((mergedRow: {}) => { - assert.deepStrictEqual(mergedRow, rows.expected); + assert.deepStrictEqual(mergedRow, rows.expectedBool); }); mergedRows = BigQuery.mergeSchemaWithRows_( @@ -621,7 +625,7 @@ describe('BigQuery', () => { wrapIntegersObject ); mergedRows.forEach((mergedRow: {}) => { - assert.deepStrictEqual(mergedRow, rows.expected); + assert.deepStrictEqual(mergedRow, rows.expectedObj); }); }); }); @@ -826,7 +830,7 @@ describe('BigQuery', () => { const INPUT_STRING = '100'; it('should call through to the static method', () => { - const fakeInt = new BigQueryInt('100'); + const fakeInt = new BigQueryInt(INPUT_STRING); sandbox .stub(BigQuery, 'int') @@ -845,9 +849,9 @@ describe('BigQuery', () => { describe('BigQueryInt', () => { it('should store the stringified value', () => { - const value = 10; - const int = new BigQueryInt(value); - assert.strictEqual(int.value, value.toString()); + const INPUT_NUM = 100; + const int = new BigQueryInt(INPUT_NUM); + assert.strictEqual(int.value, INPUT_NUM.toString()); }); describe('valueOf', () => { @@ -880,7 +884,11 @@ describe('BigQuery', () => { it('should throw if integerTypeCastOptions is provided but integerTypeCastFunction is not', () => { assert.throws( - () => new BigQueryInt(valueObject, {}).valueOf(), + () => + new BigQueryInt( + valueObject, + {} as IntegerTypeCastOptions + ).valueOf(), /integerTypeCastFunction is not a function or was not provided\./ ); }); From f275d582c334a4fbea5ff3502536b5a8256e9918 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 12 Oct 2020 09:29:27 -0700 Subject: [PATCH 05/12] add BigQueryInt type detection and tests --- src/bigquery.ts | 24 ++++++++++++++++-------- src/job.ts | 2 +- src/table.ts | 3 +-- test/bigquery.ts | 34 +++++++++++++++++++++++++++++++--- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index cecfe85a..47c7b512 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -423,7 +423,7 @@ export class BigQuery extends common.Service { * 'INT64' type in {@link BigQueryInt} objects. * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. * If an `object`, this will return a value returned by - * `wrapNumbers.integerTypeCastFunction`. + * `wrapIntegers.integerTypeCastFunction`. * Please see {@link IntegerTypeCastOptions} for options descriptions. * @param {array} selectedFields List of fields to return. * If unspecified, all fields are returned. @@ -814,9 +814,9 @@ export class BigQuery extends common.Service { * integerTypeCastFunction: customTypeCast * }; * - * const bigQueryInt = bigquery.int(largeIntegerValue, wrapIntegers); + * const bqInteger = bigquery.int(largeIntegerValue, wrapIntegers); * - * const wrappedValue = bigQueryInt.valueOf(); + * const customValue = bqInteger.valueOf(); */ static int( value: string | number | IntegerTypeCastValue, @@ -982,6 +982,8 @@ export class BigQuery extends common.Service { typeName = 'BYTES'; } else if (value instanceof Big) { typeName = 'NUMERIC'; + } else if (value instanceof BigQueryInt) { + typeName = 'INT64'; } else if (Array.isArray(value)) { if (value.length === 0) { throw new Error( @@ -1033,7 +1035,7 @@ export class BigQuery extends common.Service { * @see [Jobs.query API Reference Docs (see `queryParameters`)]{@link https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#request-body} * * @param {*} value The value. - * @param {string | ProvidedTypeStruct | ProvidedTypeArray} providedType Provided + * @param {string|ProvidedTypeStruct|ProvidedTypeArray} providedType Provided * query parameter type. * @returns {object} A properly-formed `queryParameter` object. */ @@ -1045,7 +1047,6 @@ export class BigQuery extends common.Service { if (is.date(value)) { value = BigQuery.timestamp(value as Date); } - let parameterType: bigquery.IQueryParameterType; if (providedType) { parameterType = BigQuery.getTypeDescriptorFromProvidedType_(providedType); @@ -1097,11 +1098,18 @@ export class BigQuery extends common.Service { // eslint-disable-next-line @typescript-eslint/no-explicit-any function getValue(value: any, type: ValueType): any { + if (value.type!) { + type = value; + } return isCustomType(type) ? value.value : value; } function isCustomType({type}: ValueType): boolean { - return type!.indexOf('TIME') > -1 || type!.indexOf('DATE') > -1; + return ( + type!.indexOf('TIME') > -1 || + type!.indexOf('DATE') > -1 || + type!.indexOf('BigQueryInt') > -1 + ); } } @@ -1808,7 +1816,7 @@ export class BigQuery extends common.Service { * of 'INT64' type in {@link BigQueryInt} objects. * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. * If an `object`, this will return a value returned by - * `wrapNumbers.integerTypeCastFunction`. + * `wrapIntegers.integerTypeCastFunction`. * Please see {@link IntegerTypeCastOptions} for options descriptions. * @param {function} [callback] The callback function. * @param {?error} callback.err An error returned while making this request @@ -2033,7 +2041,7 @@ export class BigQueryTime { * Build a BigQueryInt object. For long integers, a string can be provided. * * @class - * @param {string|number|IntegerTypeCastValue} value The integer value. + * @param {string|number|IntegerTypeCastValue} value The 'INT64' value. * @param {object} [typeCastOptions] Configuration to convert * values of 'INT64' type to a custom value. Must provide an * `integerTypeCastFunction` to handle conversion. diff --git a/src/job.ts b/src/job.ts index feb877ce..7501239f 100644 --- a/src/job.ts +++ b/src/job.ts @@ -344,7 +344,7 @@ class Job extends Operation { * of 'INT64' type in {@link BigQueryInt} objects. * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. * If an `object`, this will return a value returned by - * `wrapNumbers.integerTypeCastFunction`. + * `wrapIntegers.integerTypeCastFunction`. * @param {QueryResultsCallback|ManualQueryResultsCallback} [callback] The * callback function. If `autoPaginate` is set to false a * {@link ManualQueryResultsCallback} should be used. diff --git a/src/table.ts b/src/table.ts index 049b450e..e8e62edb 100644 --- a/src/table.ts +++ b/src/table.ts @@ -1628,7 +1628,7 @@ class Table extends common.ServiceObject { * of 'INT64' type in {@link BigQueryInt} objects. * If a `boolean`, this will wrap values in {@link BigQueryInt} objects. * If an `object`, this will return a value returned by - * `wrapNumbers.integerTypeCastFunction`. + * `wrapIntegers.integerTypeCastFunction`. * @param {function} [callback] The callback function. * @param {?error} callback.err An error returned while making this request * @param {array} callback.rows The table data from specified set of rows. @@ -1688,7 +1688,6 @@ class Table extends common.ServiceObject { callback!(err, null, null, resp); return; } - rows = BigQuery.mergeSchemaWithRows_( this.metadata.schema, rows || [], diff --git a/test/bigquery.ts b/test/bigquery.ts index 9b8125a7..e2791524 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -954,7 +954,7 @@ describe('BigQuery', () => { ); }); - it('should custom-cast integerValue when integerTypeCastFunction is provided', () => { + it('should custom-cast value when integerTypeCastFunction is provided', () => { const stub = sinon.stub(); new BigQueryInt(valueObject, { @@ -963,7 +963,7 @@ describe('BigQuery', () => { assert.ok(stub.calledOnce); }); - it('should custom-cast integerValue if in `fields` specified by user', () => { + it('should custom-cast value if in `fields` specified by user', () => { const stub = sinon.stub(); Object.assign(valueObject, { @@ -977,7 +977,7 @@ describe('BigQuery', () => { assert.ok(stub.calledOnce); }); - it('should not custom-cast integerValue if not in `fields` specified by user', () => { + it('should not custom-cast value if not in `fields` specified by user', () => { const stub = sinon.stub(); Object.assign(valueObject, { @@ -1054,6 +1054,10 @@ describe('BigQuery', () => { BigQuery.getTypeDescriptorFromValue_(new Big('1.1')).type, 'NUMERIC' ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(new BigQueryInt('100')).type, + 'INT64' + ); }); it('should return correct type for an array', () => { @@ -1267,6 +1271,30 @@ describe('BigQuery', () => { assert.deepStrictEqual(parameterValue.arrayValues, times); }); + it('should locate the value on BigQueryInt objects', () => { + const int = new BigQueryInt(100); + + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ + type: 'INT64', + }); + + const queryParameter = BigQuery.valueToQueryParameter_(int); + assert.strictEqual(queryParameter.parameterValue.value, int.value); + }); + + it('should locate the value on nested BigQueryInt objects', () => { + const ints = [new BigQueryInt('100')]; + const expected = [{value: '100'}]; + + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ + type: 'ARRAY', + arrayType: {type: 'INT64'}, + }); + + const {parameterValue} = BigQuery.valueToQueryParameter_(ints); + assert.deepStrictEqual(parameterValue.arrayValues, expected); + }); + it('should format an array', () => { const array = [1]; From e012d19fb33cbb14bc9b61c4a1b961fb394cd902 Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 12 Oct 2020 11:17:35 -0700 Subject: [PATCH 06/12] update test --- test/bigquery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bigquery.ts b/test/bigquery.ts index e2791524..5733f357 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -1055,7 +1055,7 @@ describe('BigQuery', () => { 'NUMERIC' ); assert.strictEqual( - BigQuery.getTypeDescriptorFromValue_(new BigQueryInt('100')).type, + BigQuery.getTypeDescriptorFromValue_(bq.int('100')).type, 'INT64' ); }); From 35ed3021f20488c2e85f446ab04ad344651d3adc Mon Sep 17 00:00:00 2001 From: steffnay Date: Wed, 14 Oct 2020 07:25:49 -0700 Subject: [PATCH 07/12] add BigQueryInt to Table.encodevalue_ --- src/table.ts | 1 + test/table.ts | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/table.ts b/src/table.ts index e8e62edb..66f7d47e 100644 --- a/src/table.ts +++ b/src/table.ts @@ -467,6 +467,7 @@ class Table extends common.ServiceObject { const customTypeConstructorNames = [ 'BigQueryDate', 'BigQueryDatetime', + 'BigQueryInt', 'BigQueryTime', 'BigQueryTimestamp', 'Geography', diff --git a/test/table.ts b/test/table.ts index 56a9fde7..a83e028d 100644 --- a/test/table.ts +++ b/test/table.ts @@ -360,16 +360,24 @@ describe('BigQuery/Table', () => { this.value = value; } } + class BigQueryInt { + value: {}; + constructor(value: {}) { + this.value = value; + } + } const date = new BigQueryDate('date'); const datetime = new BigQueryDatetime('datetime'); const time = new BigQueryTime('time'); const timestamp = new BigQueryTimestamp('timestamp'); + const integer = new BigQueryInt('integer'); assert.strictEqual(Table.encodeValue_(date), 'date'); assert.strictEqual(Table.encodeValue_(datetime), 'datetime'); assert.strictEqual(Table.encodeValue_(time), 'time'); assert.strictEqual(Table.encodeValue_(timestamp), 'timestamp'); + assert.strictEqual(Table.encodeValue_(integer), 'integer'); }); it('should properly encode arrays', () => { From d3900a7b4e2fcbcfdad1209d9155e67ec37cb0b1 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Mon, 19 Oct 2020 10:40:48 -0700 Subject: [PATCH 08/12] Update src/bigquery.ts Co-authored-by: Stephen --- src/bigquery.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 47c7b512..d641ddf8 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -810,7 +810,9 @@ export class BigQuery extends common.Service { * return value.split(''); * }; * - * const wrapIntegers = { + * const options = { + * integerTypeCastFunction: value => value.split(), + * }; * integerTypeCastFunction: customTypeCast * }; * From ac9362820bf1993a1094c6e25e321ad9fa4b3c5b Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Mon, 19 Oct 2020 10:42:11 -0700 Subject: [PATCH 09/12] Update src/bigquery.ts Co-authored-by: Stephen --- src/bigquery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index d641ddf8..2ae2186b 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -884,7 +884,7 @@ export class BigQuery extends common.Service { "To prevent this error, please consider passing 'options.wrapNumbers' as\n" + '{\n' + ' integerTypeCastFunction: provide \n' + - ' properties: optionally specify property name(s) to be custom casted\n' + + ' fields: optionally specify field name(s) to be custom casted\n' + '}\n' ); } From 6938e223cd4f6b47268e866f05c391347f1ad34a Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 19 Oct 2020 10:45:44 -0700 Subject: [PATCH 10/12] update queryAsStream and tests --- src/bigquery.ts | 28 ++++++++++++++++++++++------ test/bigquery.ts | 17 +++++++++-------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 47c7b512..4113080a 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -99,6 +99,9 @@ export type Query = JobRequest & { }; export type QueryOptions = QueryResultsOptions; +export type QueryStreamOptions = { + wrapIntegers?: boolean | IntegerTypeCastOptions; +}; export type DatasetResource = bigquery.IDataset; export type ValueType = bigquery.IQueryParameterType; @@ -817,6 +820,7 @@ export class BigQuery extends common.Service { * const bqInteger = bigquery.int(largeIntegerValue, wrapIntegers); * * const customValue = bqInteger.valueOf(); + * // customValue is the value returned from your `integerTypeCastFunction`. */ static int( value: string | number | IntegerTypeCastValue, @@ -882,7 +886,7 @@ export class BigQuery extends common.Service { "To prevent this error, please consider passing 'options.wrapNumbers' as\n" + '{\n' + ' integerTypeCastFunction: provide \n' + - ' properties: optionally specify property name(s) to be custom casted\n' + + ' fields: optionally specify field name(s) to be custom casted\n' + '}\n' ); } @@ -1929,14 +1933,26 @@ export class BigQuery extends common.Service { * * @private */ - queryAsStream_(query: Query, callback?: SimpleQueryRowsCallback) { + queryAsStream_( + query: Query, + optionsOrCallback?: QueryStreamOptions, + cb?: SimpleQueryRowsCallback + ) { + let options = + typeof optionsOrCallback === 'object' ? optionsOrCallback : {}; + const callback = + typeof optionsOrCallback === 'function' ? optionsOrCallback : cb; + + options = query.job + ? extend(query, options) + : extend(options, {autoPaginate: false}); + if (query.job) { - query.job.getQueryResults(query, callback as QueryRowsCallback); + query.job!.getQueryResults(options, callback as QueryRowsCallback); return; } - const wrapIntegers = query.wrapIntegers ? query.wrapIntegers : false; - delete query.wrapIntegers; - this.query(query, {autoPaginate: false, wrapIntegers}, callback); + + this.query(query, options, callback); } } diff --git a/test/bigquery.ts b/test/bigquery.ts index 5733f357..0ff17685 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -877,7 +877,7 @@ describe('BigQuery', () => { "To prevent this error, please consider passing 'options.wrapNumbers' as\n" + '{\n' + ' integerTypeCastFunction: provide \n' + - ' properties: optionally specify property name(s) to be custom casted\n' + + ' fields: optionally specify field name(s) to be custom casted\n' + '}\n' ); }; @@ -2577,7 +2577,7 @@ describe('BigQuery', () => { assert( queryStub.calledOnceWithExactly( query, - {autoPaginate: false, wrapIntegers: false}, + {autoPaginate: false}, sinon.match.func ) ); @@ -2597,18 +2597,19 @@ describe('BigQuery', () => { it('should pass wrapIntegers if supplied', done => { const statement = 'SELECT'; - const wrapIntegers = { - integerValue: 100, - }; const query = { query: statement, - wrapIntegers, }; - bq.queryAsStream_(query, done); + const options = { + wrapIntegers: { + integerValue: 100, + }, + }; + bq.queryAsStream_(query, options, done); assert( queryStub.calledOnceWithExactly( query, - {autoPaginate: false, wrapIntegers}, + {autoPaginate: false, wrapIntegers: options.wrapIntegers}, sinon.match.func ) ); From b7af89d9b8461d7dba0543ef1918d73d9d5390cc Mon Sep 17 00:00:00 2001 From: steffnay Date: Mon, 19 Oct 2020 11:58:31 -0700 Subject: [PATCH 11/12] fix comment --- src/bigquery.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 23b9c540..6278409a 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -809,17 +809,11 @@ export class BigQuery extends common.Service { * * const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1; * - * customTypeCast = (value) => { - * return value.split(''); - * }; - * * const options = { * integerTypeCastFunction: value => value.split(), * }; - * integerTypeCastFunction: customTypeCast - * }; * - * const bqInteger = bigquery.int(largeIntegerValue, wrapIntegers); + * const bqInteger = bigquery.int(largeIntegerValue, options); * * const customValue = bqInteger.valueOf(); * // customValue is the value returned from your `integerTypeCastFunction`. From bc1a431efd97b487d311794042bf2f0747b459ef Mon Sep 17 00:00:00 2001 From: steffnay Date: Wed, 28 Oct 2020 08:47:20 -0700 Subject: [PATCH 12/12] replace overwritten line --- src/bigquery.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bigquery.ts b/src/bigquery.ts index 49491774..4f9516e5 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -1104,6 +1104,7 @@ export class BigQuery extends common.Service { // eslint-disable-next-line @typescript-eslint/no-explicit-any private static _getValue(value: any, type: ValueType): any { + if (value.type!) type = value; return BigQuery._isCustomType(type) ? value.value : value; }