Skip to content
This repository has been archived by the owner on Jun 24, 2024. It is now read-only.

Commit

Permalink
chore: switch from google-auto-auth to google-auth-library (#59)
Browse files Browse the repository at this point in the history
  • Loading branch information
@JustinBeckwith @stephenplusplus
JustinBeckwith authored and stephenplusplus committed May 3, 2018
1 parent 9976045 commit fa4b777
Show file tree
Hide file tree
Showing 4 changed files with 186 additions and 210 deletions.
5 changes: 3 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"gcs-upload": "build/src/cli.js"
},
"scripts": {
"test": "mocha build/test -r source-map-support/register",
"test": "mocha build/test -r source-map-support/register --timeout 4000",
"check": "gts check",
"clean": "gts clean",
"compile": "tsc -p .",
Expand All @@ -33,8 +33,9 @@
"author": "Stephen Sawchuk <[email protected]>",
"license": "MIT",
"dependencies": {
"axios": "^0.18.0",
"configstore": "^3.1.2",
"google-auto-auth": "^0.10.0",
"google-auth-library": "^1.4.0",
"pumpify": "^1.4.0",
"request": "^2.85.0",
"stream-events": "^1.0.3"
Expand Down
147 changes: 69 additions & 78 deletions src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
import {AxiosError, AxiosRequestConfig, AxiosResponse} from 'axios';
import * as ConfigStore from 'configstore';
import * as crypto from 'crypto';
import {GoogleAuth, GoogleAuthOptions} from 'google-auth-library';
import * as Pumpify from 'pumpify';
import * as r from 'request';
import {Duplex, PassThrough, Readable, Stream} from 'stream';
import * as streamEvents from 'stream-events';

const googleAuth = require('google-auto-auth');
import * as Pumpify from 'pumpify';

// tslint:disable-next-line no-any
export type RequestBody = any;
export type RequestResponse = r.Response;
export type RequestResponse = AxiosResponse;
export type Request = r.Request;
export type RequestOptions = r.OptionsWithUri;
export type RequestOptions = AxiosRequestConfig;
export type RequestCallback =
(err: Error|null, response?: r.Response, body?: RequestBody) => void;
(err: Error|null, response?: AxiosResponse, body?: RequestBody) => void;
export type AuthorizeRequestCallback =
(err: Error|null, authorizedReqOpts: RequestOptions) => void;

Expand All @@ -30,13 +30,6 @@ const wrapError = (message: string, err: Error) => {

export type CreateUriCallback = (err: Error|null, uri?: string) => void;

export interface AuthClient {
authorizeRequest:
(reqOpts: r.OptionsWithUri,
callback: (err: Error|null, authorizedReqOpts: RequestOptions) =>
void) => void;
}

export interface Encryption {
key: {};
hash: {};
Expand All @@ -52,12 +45,12 @@ export interface UploadConfig {
* The name of the destination file.
*/
file: string;
authConfig?: {scopes?: string[];};
authConfig?: GoogleAuthOptions;
/**
* If you want to re-use an auth client from google-auto-auth, pass an
* instance here.
*/
authClient?: AuthClient;
authClient?: GoogleAuth;

/**
* This will cause the upload to fail if the current generation of the remote
Expand Down Expand Up @@ -143,7 +136,7 @@ export class Upload extends Pumpify {
bucket: string;
file: string;
authConfig?: {scopes?: string[];};
authClient: AuthClient;
authClient: GoogleAuth;
generation?: number;
key?: string|Buffer;
kmsKeyName?: string;
Expand Down Expand Up @@ -178,7 +171,7 @@ export class Upload extends Pumpify {
cfg.authConfig = cfg.authConfig || {};
cfg.authConfig.scopes =
['https://www.googleapis.com/auth/devstorage.full_control'];
this.authClient = cfg.authClient || googleAuth(cfg.authConfig);
this.authClient = cfg.authClient || new GoogleAuth(cfg.authConfig);

this.bucket = cfg.bucket;
this.file = cfg.file;
Expand Down Expand Up @@ -235,9 +228,9 @@ export class Upload extends Pumpify {

const reqOpts: RequestOptions = {
method: 'POST',
uri: [BASE_URI, this.bucket, 'o'].join('/'),
qs: {name: this.file, uploadType: 'resumable'},
json: metadata,
url: [BASE_URI, this.bucket, 'o'].join('/'),
params: {name: this.file, uploadType: 'resumable'},
data: metadata,
headers: {}
};

Expand All @@ -250,15 +243,15 @@ export class Upload extends Pumpify {
}

if (typeof this.generation !== 'undefined') {
reqOpts.qs.ifGenerationMatch = this.generation;
reqOpts.params.ifGenerationMatch = this.generation;
}

if (this.kmsKeyName) {
reqOpts.qs.kmsKeyName = this.kmsKeyName;
reqOpts.params.kmsKeyName = this.kmsKeyName;
}

if (this.predefinedAcl) {
reqOpts.qs.predefinedAcl = this.predefinedAcl;
reqOpts.params.predefinedAcl = this.predefinedAcl;
}

if (this.origin) {
Expand Down Expand Up @@ -287,11 +280,11 @@ export class Upload extends Pumpify {
private startUploading() {
const reqOpts = {
method: 'PUT',
uri: this.uri,
url: this.uri,
headers: {
'Content-Range': 'bytes ' + this.offset + '-*/' + this.contentLength
}
} as r.OptionsWithUri;
};

const bufferStream = this.bufferStream = new PassThrough();
const offsetStream = this.offsetStream =
Expand Down Expand Up @@ -363,16 +356,16 @@ export class Upload extends Pumpify {
private getAndSetOffset(callback: () => void) {
const opts = {
method: 'PUT',
uri: this.uri,
url: this.uri,
headers: {'Content-Length': 0, 'Content-Range': 'bytes */*'}
} as r.OptionsWithUri;
};
this.makeRequest(opts, (err, resp) => {
if (err) {
// we don't return a 404 to the user if they provided the resumable
// URI. if we're just using the configstore file to tell us that this
// file exists, and it turns out that it doesn't (the 404), that's
// probably stale config data.
if (resp && resp.statusCode === 404 && !this.uriProvidedManually) {
if (resp && resp.status === 404 && !this.uriProvidedManually) {
return this.restart();
}

Expand All @@ -381,14 +374,14 @@ export class Upload extends Pumpify {
// https://github.com/stephenplusplus/gcs-resumable-upload/issues/15
// -
// https://github.com/stephenplusplus/gcs-resumable-upload/pull/16#discussion_r80363774
if (resp && resp.statusCode === TERMINATED_UPLOAD_STATUS_CODE) {
if (resp && resp.status === TERMINATED_UPLOAD_STATUS_CODE) {
return this.restart();
}

return this.destroy(err);
}

if (resp && resp.statusCode === RESUMABLE_INCOMPLETE_STATUS_CODE) {
if (resp && resp.status === RESUMABLE_INCOMPLETE_STATUS_CODE) {
if (resp.headers.range) {
const range = resp.headers.range as string;
this.offset = Number(range.split('-')[1]) + 1;
Expand All @@ -411,62 +404,52 @@ export class Upload extends Pumpify {
}

if (this.userProject) {
reqOpts.qs = reqOpts.qs || {};
reqOpts.qs.userProject = this.userProject;
reqOpts.params = reqOpts.params || {};
reqOpts.params.userProject = this.userProject;
}

this.authClient.authorizeRequest(reqOpts, (err, authorizedReqOpts) => {
if (err) {
err = wrapError('Could not authenticate request', err);
return callback(err, null!, null);
}

request(authorizedReqOpts, (err, resp, body) => {
if (err) {
return callback(err, resp, body);
}

if (body && body.error) {
return callback(body.error, resp, body);
}

const nonSuccess =
Math.floor(resp.statusCode / 100) !== 2; // 200-299 status code
if (nonSuccess &&
resp.statusCode !== RESUMABLE_INCOMPLETE_STATUS_CODE) {
return callback(new Error(body), resp, body);
}
reqOpts.validateStatus = (status: number) => {
return (status >= 200 && status < 300) ||
status === RESUMABLE_INCOMPLETE_STATUS_CODE;
};

callback(null, resp, body);
});
});
this.authClient.request(reqOpts).then(
r => {
return callback(null, r, r.data);
},
(err: AxiosError) => {
const body = err.response ? err.response.data : undefined;
const e = (body && body.error) ? body.error : err;
return callback(e, err.response, body);
});
}

private getRequestStream(
reqOpts: RequestOptions, callback: (requestStream: Request) => void) {
if (this.userProject) {
reqOpts.qs = reqOpts.qs || {};
reqOpts.qs.userProject = this.userProject;
reqOpts.params = reqOpts.params || {};
reqOpts.params.userProject = this.userProject;
}

this.authClient.authorizeRequest(reqOpts, (err, authorizedReqOpts) => {
if (err) {
return this.destroy(wrapError('Could not authenticate request', err));
}

const requestStream = request(authorizedReqOpts);
requestStream.on('error', this.destroy.bind(this));
requestStream.on('response', this.onResponse.bind(this));
requestStream.on('complete', (resp) => {
const body = resp.body;
if (body && body.error) this.destroy(body.error);
});

// this makes the response body come back in the response (weird?)
requestStream.callback = () => {};

callback(requestStream);
});
this.authClient.authorizeRequest(reqOpts)
.then(opts => {
const authorizedReqOpts = axiosToRequest(reqOpts);
const requestStream = request(authorizedReqOpts);
requestStream.on('error', this.destroy.bind(this));
requestStream.on('response', this.onResponse.bind(this));
requestStream.on('complete', (resp) => {
const body = resp.body;
if (body && body.error) this.destroy(body.error);
});

// this makes the response body come back in the response (weird?)
requestStream.callback = () => {};

callback(requestStream);
})
.catch(err => {
return this.destroy(wrapError('Could not authenticate request', err));
});
}

private restart() {
Expand Down Expand Up @@ -498,7 +481,7 @@ export class Upload extends Pumpify {
* @return {bool} is the request good?
*/
private onResponse(resp: RequestResponse) {
if (resp.statusCode === 404) {
if (resp.status === 404) {
if (this.numRetries < RETRY_LIMIT) {
this.numRetries++;
this.startUploading();
Expand All @@ -508,7 +491,7 @@ export class Upload extends Pumpify {
return false;
}

if (resp.statusCode > 499 && resp.statusCode < 600) {
if (resp.status > 499 && resp.status < 600) {
if (this.numRetries < RETRY_LIMIT) {
const randomMs = Math.round(Math.random() * 1000);
const waitTime = Math.pow(2, this.numRetries) * 1000 + randomMs;
Expand All @@ -527,6 +510,14 @@ export class Upload extends Pumpify {
}
}

function axiosToRequest(opts: AxiosRequestConfig): r.OptionsWithUri {
const reqOpts = opts as r.OptionsWithUri;
reqOpts.qs = opts.params;
reqOpts.json = opts.data;
reqOpts.uri = opts.url as string;
return reqOpts;
}

export function upload(cfg: UploadConfig) {
return new Upload(cfg);
}
Expand Down
12 changes: 12 additions & 0 deletions test/fixtures/keys.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"type": "service_account",
"project_id": "project-id",
"private_key_id": "12345",
"private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC5z21IhrvlHBj7\nifRhobA9ibn25Od7DpE5OauGmqy7B+A9LQOsk1ZujAFdHItnBPcjihSVHpiYxf1a\nLpFbM8z/hRvDvYS3Hs1pyRejmpGiznoOjCyUf6Wv3T1xKelbgn0twHHjqD1o0xzW\njyUILl7yuCbsAf8QlsV6ewS3IqO3i5A9RNHfKjeap8e6A7U3s9QBtR58RrxaMQpM\nz72gw7yOdJRfElkerQfyZTbtu/EBfE6CcskOyoMoRN3YgkjQqLMr1yVdL5/phEcQ\n5hpbDN5lrafGHN7FUtsrMge2iIuIYFfWQUTqu7HtnNXVmwj1LJNq5WeI1iInWaGz\nb7c1rUT9AgMBAAECggEAEB0FicqVX3L7qk9LsBkeItgKFnfB/eaaKsTuM7K/fqCv\njjPpzlIgprQ20g+i+dYbuytC9Fjo5tFp/SNuBji3Ha7kuil56Yoe9NOJh0M6P0zP\nQj+0W1Rj1p0bB5tDhLoLh6eEDjgNde+zioUeCFhCck4MogmHnbVVfspNnba/99oD\nl36heAioqj/KODdkbQ83+ByiH+3BzqblqJ4TR/3y0wOUXtlQvCHko1qximJFIM0z\n3TNoPiit74hTiFFOYfJyHpmRsiEJ5FUUImkmCJz2gk4fbpafKrgxxOMo1m7GqlsE\nE+ybHxyAq61HYbZOoUOO8B4md1/52QXP7DgPvV7JyQKBgQD+JS5nsR4TXRl61c9G\nNxoPW9yCMCoarIjkpyPmhh0uJ7y68cj9wHFgX6ATi1QuTnG9BzJ4z27PMgvv70N+\nAK6k74sdIT2ts8wYsD8H0UyuxDxeKiAnb2JW2f5GTcXNmELQi6rKkMNMoS8jv00d\ngzLCV7UbCbdf+ng9uRPs+Fvk9wKBgQC7KpNaeYFf5dmIYRWQhlZWBRoftdm1ROH/\n5GJsURkzlEjUH1g1y9eAigBn5I+Z9hylX2q1vHLpUHqONWwDz8oQ1L1o2iLz+tkp\nkNoaLSAb9uCl6t8tpqCG2dqUrxOmy1+xj3G8KI8XuYb+IwVSy6KK2df8fWN4d+i0\ng+TBb75MqwKBgEezwcXriKq554hqblJHFYkjx7DLWfWwm+a26UAOsojlGTA9KxG8\ni8A++nDJLHTsGNbWAv1muMKoQgntnUMdeih6lOshB7/MLFcC0qWn/VSJdOa0R+IY\nYMxUMJMxOg9pV+BypzsDYLZr+1rAjEc5TsbZ6/S25w+jIO15HBANeg+9AoGAZulz\nGkVDCLq2UJGpLM1gvW2Svqrb6RrV9UDbiVlSNRUssk4Fz5akiM3YiUeYWfyEJb4A\nS6sxt+4DZRwkpzfikDyZZQTEQUjFjWBTPB9hz16AiVpKmqxLCbrRv/1AHe8nT9di\nnyXiABaIDkatT6geWKCNbQx43C16a382EdJiXX8CgYEAqyAS2xuDi2+uoljRm1Bp\naz7Q2UBtBbcBr/CQmagEacWPXsSyCL6EySOH0e985k7ABZiW+AzWlOwKS5WMWAIb\nkncmxP0SU6WQDWl8xGbXAQ8Dw+HTu5G1n0vrl1rRO5FPwRs3pbV94ML+d5eoai6D\njHs1asOGIpdQ3OGpBpNRub0=\n-----END PRIVATE KEY-----\n",
"client_email": "[email protected]",
"client_id": "12345",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://accounts.google.com/o/oauth2/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/el-gato%40el-gato.iam.gserviceaccount.com"
}
Loading

0 comments on commit fa4b777

Please sign in to comment.