Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: optionally wrap INT64 in BigQueryInt or provide a custom INT64 value type cast options #873

Merged
merged 19 commits into from
Oct 28, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
225 changes: 217 additions & 8 deletions src/bigquery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,13 @@ export type Query = JobRequest<bigquery.IJobConfigurationQuery> & {
maxResults?: number;
timeoutMs?: number;
pageToken?: string;
wrapIntegers?: boolean | IntegerTypeCastOptions;
};

export type QueryOptions = QueryResultsOptions;
export type QueryStreamOptions = {
wrapIntegers?: boolean | IntegerTypeCastOptions;
};
export type DatasetResource = bigquery.IDataset;
export type ValueType = bigquery.IQueryParameterType;

Expand Down Expand Up @@ -171,6 +175,16 @@ export interface BigQueryOptions extends common.GoogleAuthOptions {
apiEndpoint?: string;
}

export interface IntegerTypeCastOptions {
integerTypeCastFunction: Function;
fields?: string | string[];
}

export type IntegerTypeCastValue = {
integerValue: string | number;
schemaFieldName?: string;
};

export const PROTOCOL_REGEX = /^(\w*):\/\//;

/**
Expand Down Expand Up @@ -408,13 +422,20 @@ export class BigQuery extends common.Service {
*
* @param {object} schema
* @param {array} rows
* @param {boolean|IntegerTypeCastOptions} wrapIntegers Wrap values of
* 'INT64' type in {@link BigQueryInt} objects.
* If a `boolean`, this will wrap values in {@link BigQueryInt} objects.
* If an `object`, this will return a value returned by
* `wrapIntegers.integerTypeCastFunction`.
* Please see {@link IntegerTypeCastOptions} for options descriptions.
* @param {array} selectedFields List of fields to return.
* If unspecified, all fields are returned.
* @returns {array} Fields using their matching names from the table's schema.
*/
static mergeSchemaWithRows_(
schema: TableSchema | TableField,
rows: TableRow[],
wrapIntegers: boolean | IntegerTypeCastOptions,
selectedFields?: string[]
) {
if (selectedFields && selectedFields!.length > 0) {
Expand Down Expand Up @@ -444,10 +465,10 @@ export class BigQuery extends common.Service {
let value = field.v;
if (schemaField.mode === 'REPEATED') {
value = (value as TableRowField[]).map(val => {
return convert(schemaField, val.v, selectedFields);
return convert(schemaField, val.v, wrapIntegers, selectedFields);
});
} else {
value = convert(schemaField, value, selectedFields);
value = convert(schemaField, value, wrapIntegers, selectedFields);
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const fieldObject: any = {};
Expand All @@ -460,6 +481,7 @@ export class BigQuery extends common.Service {
schemaField: TableField,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
value: any,
wrapIntegers: boolean | IntegerTypeCastOptions,
selectedFields?: string[]
) {
if (is.null(value)) {
Expand All @@ -483,7 +505,14 @@ export class BigQuery extends common.Service {
}
case 'INTEGER':
case 'INT64': {
value = Number(value);
value = wrapIntegers
? typeof wrapIntegers === 'object'
? BigQuery.int(
{integerValue: value, schemaFieldName: schemaField.name},
wrapIntegers
).valueOf()
: BigQuery.int(value)
: Number(value);
break;
}
case 'NUMERIC': {
Expand All @@ -494,6 +523,7 @@ export class BigQuery extends common.Service {
value = BigQuery.mergeSchemaWithRows_(
schemaField,
value,
wrapIntegers,
selectedFields
).pop();
break;
Expand Down Expand Up @@ -765,6 +795,43 @@ export class BigQuery extends common.Service {
return BigQuery.timestamp(value);
}

/**
* A BigQueryInt wraps 'INT64' values. Can be used to maintain precision.
*
* @method BigQuery#int
* @param {string|number|IntegerTypeCastValue} value The INT64 value to convert.
* @param {IntegerTypeCastOptions} typeCastOptions Configuration to convert
* value. Must provide an `integerTypeCastFunction` to handle conversion.
*
* @example
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
*
* const largeIntegerValue = Number.MAX_SAFE_INTEGER + 1;
*
* const options = {
* integerTypeCastFunction: value => value.split(),
* };
*
* const bqInteger = bigquery.int(largeIntegerValue, options);
*
* const customValue = bqInteger.valueOf();
steffnay marked this conversation as resolved.
Show resolved Hide resolved
* // customValue is the value returned from your `integerTypeCastFunction`.
*/
static int(
value: string | number | IntegerTypeCastValue,
typeCastOptions?: IntegerTypeCastOptions
) {
return new BigQueryInt(value, typeCastOptions);
}

int(
value: string | number | IntegerTypeCastValue,
typeCastOptions?: IntegerTypeCastOptions
) {
return BigQuery.int(value, typeCastOptions);
}

/**
* A geography value represents a surface area on the Earth
* in Well-known Text (WKT) format.
Expand Down Expand Up @@ -797,6 +864,31 @@ export class BigQuery extends common.Service {
return BigQuery.geography(value);
}

/**
* Convert an INT64 value to Number.
*
* @private
* @param {object} value The INT64 value to convert.
*/
static decodeIntegerValue_(value: IntegerTypeCastValue) {
const num = Number(value.integerValue);
if (!Number.isSafeInteger(num)) {
throw new Error(
'We attempted to return all of the numeric values, but ' +
(value.schemaFieldName ? value.schemaFieldName + ' ' : '') +
'value ' +
value.integerValue +
" is out of bounds of 'Number.MAX_SAFE_INTEGER'.\n" +
"To prevent this error, please consider passing 'options.wrapNumbers' as\n" +
'{\n' +
' integerTypeCastFunction: provide <your_custom_function>\n' +
' fields: optionally specify field name(s) to be custom casted\n' +
'}\n'
);
}
return num;
}

/**
* Return a value's provided type.
*
Expand Down Expand Up @@ -890,6 +982,8 @@ export class BigQuery extends common.Service {
typeName = 'BYTES';
} else if (value instanceof Big) {
typeName = 'NUMERIC';
} else if (value instanceof BigQueryInt) {
typeName = 'INT64';
} else if (value instanceof Geography) {
typeName = 'GEOGRAPHY';
} else if (Array.isArray(value)) {
Expand Down Expand Up @@ -943,6 +1037,8 @@ export class BigQuery extends common.Service {
* @see [Jobs.query API Reference Docs (see `queryParameters`)]{@link https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query#request-body}
*
* @param {*} value The value.
* @param {string|ProvidedTypeStruct|ProvidedTypeArray} providedType Provided
* query parameter type.
* @returns {object} A properly-formed `queryParameter` object.
*/
static valueToQueryParameter_(
Expand All @@ -953,7 +1049,6 @@ export class BigQuery extends common.Service {
if (is.date(value)) {
value = BigQuery.timestamp(value as Date);
}

let parameterType: bigquery.IQueryParameterType;
if (providedType) {
parameterType = BigQuery.getTypeDescriptorFromProvidedType_(providedType);
Expand Down Expand Up @@ -1009,14 +1104,16 @@ export class BigQuery extends common.Service {

// eslint-disable-next-line @typescript-eslint/no-explicit-any
private static _getValue(value: any, type: ValueType): any {
if (value.type!) type = value;
return BigQuery._isCustomType(type) ? value.value : value;
}

private static _isCustomType({type}: ValueType): boolean {
return (
type!.indexOf('TIME') > -1 ||
type!.indexOf('DATE') > -1 ||
type!.indexOf('GEOGRAPHY') > -1
type!.indexOf('GEOGRAPHY') > -1 ||
type!.indexOf('BigQueryInt') > -1
);
}

Expand Down Expand Up @@ -1719,6 +1816,12 @@ export class BigQuery extends common.Service {
* complete, in milliseconds, before returning. Default is to return
* immediately. If the timeout passes before the job completes, the
* request will fail with a `TIMEOUT` error.
* @param {boolean|IntegerTypeCastOptions} [options.wrapIntegers=false] Wrap values
* of 'INT64' type in {@link BigQueryInt} objects.
* If a `boolean`, this will wrap values in {@link BigQueryInt} objects.
* If an `object`, this will return a value returned by
* `wrapIntegers.integerTypeCastFunction`.
* Please see {@link IntegerTypeCastOptions} for options descriptions.
* @param {function} [callback] The callback function.
* @param {?error} callback.err An error returned while making this request
* @param {array} callback.rows The list of results from your query.
Expand Down Expand Up @@ -1830,12 +1933,26 @@ export class BigQuery extends common.Service {
*
* @private
*/
queryAsStream_(query: Query, callback?: SimpleQueryRowsCallback) {
queryAsStream_(
query: Query,
optionsOrCallback?: QueryStreamOptions,
cb?: SimpleQueryRowsCallback
) {
let options =
typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
const callback =
typeof optionsOrCallback === 'function' ? optionsOrCallback : cb;

options = query.job
? extend(query, options)
: extend(options, {autoPaginate: false});

if (query.job) {
query.job.getQueryResults(query, callback as QueryRowsCallback);
query.job!.getQueryResults(options, callback as QueryRowsCallback);
return;
}
this.query(query, {autoPaginate: false}, callback);

this.query(query, options, callback);
}
}

Expand All @@ -1856,6 +1973,7 @@ promisifyAll(BigQuery, {
'date',
'datetime',
'geography',
'int',
'job',
'time',
'timestamp',
Expand Down Expand Up @@ -1934,3 +2052,94 @@ export class BigQueryTime {
this.value = value as string;
}
}

/**
* Build a BigQueryInt object. For long integers, a string can be provided.
*
* @class
* @param {string|number|IntegerTypeCastValue} value The 'INT64' value.
* @param {object} [typeCastOptions] Configuration to convert
* values of 'INT64' type to a custom value. Must provide an
* `integerTypeCastFunction` to handle conversion.
* @param {function} typeCastOptions.integerTypeCastFunction A custom user
* provided function to convert value.
* @param {string|string[]} [typeCastOptions.fields] Schema field
* names to be converted using `integerTypeCastFunction`.
*
* @example
* const {BigQuery} = require('@google-cloud/bigquery');
* const bigquery = new BigQuery();
* const anInt = bigquery.int(7);
*/
export class BigQueryInt extends Number {
type: string;
value: string;
typeCastFunction?: Function;
private _schemaFieldName: string | undefined;
constructor(
value: string | number | IntegerTypeCastValue,
typeCastOptions?: IntegerTypeCastOptions
) {
super(typeof value === 'object' ? value.integerValue : value);
this._schemaFieldName =
typeof value === 'object' ? value.schemaFieldName : undefined;
this.value =
typeof value === 'object'
? value.integerValue.toString()
: value.toString();

this.type = 'BigQueryInt';

if (typeCastOptions) {
if (typeof typeCastOptions.integerTypeCastFunction !== 'function') {
throw new Error(
'integerTypeCastFunction is not a function or was not provided.'
);
}

const typeCastFields = typeCastOptions.fields
? arrify(typeCastOptions.fields)
: undefined;

let customCast = true;

if (typeCastFields) {
customCast = this._schemaFieldName
? typeCastFields.includes(this._schemaFieldName)
? true
: false
: false;
}

customCast &&
(this.typeCastFunction = typeCastOptions.integerTypeCastFunction);
}
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
valueOf(): any {
const shouldCustomCast = this.typeCastFunction ? true : false;

if (shouldCustomCast) {
try {
return this.typeCastFunction!(this.value);
} catch (error) {
error.message = `integerTypeCastFunction threw an error:\n\n - ${error.message}`;
throw error;
}
} else {
// return this.value;
return BigQuery.decodeIntegerValue_({
integerValue: this.value,
schemaFieldName: this._schemaFieldName,
});
}
}

toJSON(): Json {
return {type: this.type, value: this.value};
}
}

export interface Json {
[field: string]: string;
}
4 changes: 4 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ export {
BigQueryDateOptions,
BigQueryDatetime,
BigQueryDatetimeOptions,
BigQueryInt,
BigQueryOptions,
BigQueryTime,
BigQueryTimeOptions,
Expand All @@ -33,7 +34,10 @@ export {
GetJobsCallback,
GetJobsOptions,
GetJobsResponse,
IntegerTypeCastOptions,
IntegerTypeCastValue,
JobRequest,
Json,
PagedCallback,
PagedRequest,
PagedResponse,
Expand Down
Loading