Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add ESM tools in gax #1459

Merged
merged 33 commits into from
Sep 19, 2023
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
1b53525
feat!: publish protos.js using es6 and amd, if needed
sofisl May 18, 2023
7266376
feat: add babel transform for import.meta.url
ddelgrosso1 Jun 20, 2023
5a2916e
add transformer for esm mocking lib
ddelgrosso1 Jun 22, 2023
112b3d9
add transform for toggling isEsm flag
ddelgrosso1 Jul 19, 2023
3a3dd27
Merge branch 'main' of github.com:googleapis/gax-nodejs into esm-cjs-…
sofisl Aug 22, 2023
4c8f319
Merge remote-tracking branch 'upstream/main' into esm-cjs-tools
sofisl Aug 23, 2023
6721dd0
Merge remote-tracking branch 'upstream/updateProtosJs' into esm-cjs-t…
sofisl Aug 28, 2023
9abbdc9
build: update branch and update buildListOfProtos func
sofisl Aug 28, 2023
1416fb6
save for now
sofisl Aug 29, 2023
d63460b
feat: add babel transform for import.meta.url
ddelgrosso1 Jun 20, 2023
6f683af
add transformer for esm mocking lib
ddelgrosso1 Jun 22, 2023
c7514b4
add transform for toggling isEsm flag
ddelgrosso1 Jul 19, 2023
80de022
change renameFiles
sofisl Aug 31, 2023
c6958de
replace whole expression
sofisl Aug 31, 2023
236fdea
use callexpression
sofisl Aug 31, 2023
1840358
save for now
sofisl Aug 31, 2023
e0c1f92
move files back to .json
sofisl Sep 1, 2023
8749891
fix linter problems, update replaceImportMetaUrl tests
ddelgrosso1 Sep 1, 2023
c2bb035
add header to test/renameFile.ts
ddelgrosso1 Sep 1, 2023
422c8d8
change amd to esm
sofisl Sep 1, 2023
f64da12
merge hcanges
sofisl Sep 1, 2023
429fe53
save for now
sofisl Sep 1, 2023
06d1457
chore: update tests for esm
sofisl Sep 2, 2023
b8349d0
run lint
sofisl Sep 2, 2023
4c0a517
delete unneeded file
sofisl Sep 2, 2023
8eaac22
Merge branch 'main' into esm-cjs-tools
ddelgrosso1 Sep 7, 2023
83f0cc1
Merge branch 'esm-cjs-tools' of github.com:ddelgrosso1/gax-nodejs int…
sofisl Sep 12, 2023
54ce22e
remove path matching
sofisl Sep 12, 2023
c8eea35
Update compileProtos.ts
sofisl Sep 18, 2023
dc31cd2
Update compileProtos.ts
sofisl Sep 18, 2023
9bd5c37
Merge branch 'main' into esm-cjs-tools
sofisl Sep 18, 2023
645f063
chore: update compilePrtotos
sofisl Sep 18, 2023
3b2769d
Merge branch 'main' into esm-cjs-tools
sofisl Sep 19, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions tools/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,17 @@
"google-proto-files": "^4.0.0",
"protobufjs-cli": "1.1.2",
"rimraf": "^5.0.1",
"@babel/core": "^7.22.5",
"@babel/traverse": "^7.22.5",
"uglify-js": "^3.17.0",
"walkdir": "^0.4.0"
},
"repository": "googleapis/gax-nodejs",
"devDependencies": {
"@babel/cli": "^7.22.5",
"@babel/types": "^7.22.5",
"@types/babel__core": "^7.20.1",
"@types/babel__traverse": "^7.20.1",
"@types/mocha": "^9.0.0",
"@types/ncp": "^2.0.1",
"@types/uglify-js": "^3.17.0",
Expand Down
68 changes: 60 additions & 8 deletions tools/src/compileProtos.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,13 @@ function updateDtsTypes(dts: string, enums: Set<string>): string {
}

function fixJsFile(js: string): string {
// 1. fix protobufjs import: we don't want the libraries to
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

super nit but let's either make this "0." or increment the numbers below :)

// depend on protobufjs, so we re-export it from google-gax
js = js.replace(
'import * as $protobuf from "protobufjs/minimal"',
'import { protobufMinimal as $protobuf} from "google-gax/build/src/protobuf.js"'
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: formatting, remove space after {

);

// 1. fix protobufjs require: we don't want the libraries to
// depend on protobufjs, so we re-export it from google-gax
js = js.replace(
Expand Down Expand Up @@ -211,13 +218,19 @@ function fixDtsFile(dts: string): string {
* @param {string[]} protoJsonFiles List of JSON files to parse
* @return {Promise<string[]>} Resolves to an array of proto files.
*/
async function buildListOfProtos(protoJsonFiles: string[]): Promise<string[]> {
async function buildListOfProtos(
protoJsonFiles: string[],
esm?: boolean
): Promise<string[]> {
const result: string[] = [];
for (const file of protoJsonFiles) {
const directory = path.dirname(file);
const content = await readFile(file);
const list = JSON.parse(content.toString()).map((filePath: string) =>
path.join(directory, normalizePath(filePath))
// If we're in ESM, we're going to be in a directory level below normal
esm
? path.join(directory, '..', normalizePath(filePath))
: path.join(directory, normalizePath(filePath))
);
result.push(...list);
}
Expand All @@ -236,7 +249,8 @@ async function buildListOfProtos(protoJsonFiles: string[]): Promise<string[]> {
async function compileProtos(
rootName: string,
protos: string[],
skipJson = false
skipJson = false,
esm = false
): Promise<void> {
if (!skipJson) {
// generate protos.json file from proto list
Expand All @@ -261,7 +275,9 @@ async function compileProtos(
}

// generate protos/protos.js from protos.json
const jsOutput = path.join('protos', 'protos.js');
const jsOutput = esm
? path.join('protos', 'protos.cjs')
: path.join('protos', 'protos.js');
const pbjsArgs4js = [
'-r',
rootName,
Expand All @@ -281,9 +297,34 @@ async function compileProtos(
jsResult = fixJsFile(jsResult);
await writeFile(jsOutput, jsResult);

let jsOutputEsm;
if (esm) {
jsOutputEsm = path.join('protos', 'protos.js');
const pbjsArgs4jsEsm = [
'-r',
rootName,
'--target',
'static-module',
'-p',
'protos',
'-p',
gaxProtos,
'-o',
jsOutputEsm,
'-w',
'es6',
];
pbjsArgs4jsEsm.push(...protos);
await pbjsMain(pbjsArgs4jsEsm);

let jsResult = (await readFile(jsOutputEsm)).toString();
jsResult = fixJsFile(jsResult);
await writeFile(jsOutputEsm, jsResult);
}

// generate protos/protos.d.ts
const tsOutput = path.join('protos', 'protos.d.ts');
const pbjsArgs4ts = [jsOutput, '-o', tsOutput];
const pbjsArgs4ts = [esm ? jsOutputEsm! : jsOutput, '-o', tsOutput];
await pbtsMain(pbjsArgs4ts);

let tsResult = (await readFile(tsOutput)).toString();
Expand Down Expand Up @@ -330,27 +371,38 @@ export async function generateRootName(directories: string[]): Promise<string> {
export async function main(parameters: string[]): Promise<void> {
const protoJsonFiles: string[] = [];
let skipJson = false;
let esm = false;
const directories: string[] = [];
for (const parameter of parameters) {
if (parameter === '--skip-json') {
skipJson = true;
continue;
}
if (parameter === '--esm') {
esm = true;
continue;
}
// it's not an option so it's a directory
const directory = parameter;
directories.push(directory);
protoJsonFiles.push(...(await findProtoJsonFiles(directory)));
}
const rootName = await generateRootName(directories);
const protos = await buildListOfProtos(protoJsonFiles);
await compileProtos(rootName, protos, skipJson);
if (esm) {
const esmProtos = await buildListOfProtos(protoJsonFiles, esm);
await compileProtos(rootName, esmProtos, skipJson, esm);
}
const protos = await buildListOfProtos(protoJsonFiles, esm);
await compileProtos(rootName, protos, skipJson, esm);
}

/**
* Shows the usage information.
*/
function usage() {
console.log(`Usage: node ${process.argv[1]} [--skip-json] directory ...`);
console.log(
`Usage: node ${process.argv[1]} [--skip-json] [--esm] directory ...`
);
console.log(
`Finds all files matching ${PROTO_LIST_REGEX} in the given directories.`
);
Expand Down
64 changes: 64 additions & 0 deletions tools/src/replaceESMMockingLib.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {Visitor, types} from '@babel/core';

export interface PluginOptions {
opts?: {
fromLibName?: string;
toLibName?: string;
};
}

export default function replaceESMMockingLib(): {
visitor: Visitor<PluginOptions>;
} {
return {
visitor: {
ImportDeclaration(path, state) {
const opts = state.opts || {};
const fromLib = opts.fromLibName || 'esmock';
const toLib = opts.toLibName || 'proxyquire';
const {node} = path;

node.specifiers.forEach(spec => {
if (spec.local.name !== fromLib) {
return;
}
spec.local.name = spec.local.name.replace(fromLib, toLib);
});

if (node.source.value !== fromLib) {
return;
}
node.source.value = node.source.value.replace(fromLib, toLib);
},
CallExpression(path, state) {
const opts = state.opts || {};
const fromLib = opts.fromLibName || 'esmock';
const toLib = opts.toLibName || 'proxyquire';
const {node} = path;

if (types.isIdentifier(node.callee)) {
if (node.callee.name !== fromLib) {
return;
}

node.callee.name = toLib;
path.parentPath.replaceWith(types.expressionStatement(node));
}
},
},
};
}
60 changes: 60 additions & 0 deletions tools/src/replaceImportMetaUrl.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// eslint-disable-next-line node/no-extraneous-import
import {smart} from '@babel/template';
import {Statement} from '@babel/types';
import {Visitor} from '@babel/core';

export interface PluginOptions {
opts?: {
replacementValue?: string;
};
}

export default function replaceImportMetaUrl(): {
visitor: Visitor<PluginOptions>;
} {
return {
visitor: {
CallExpression(path, state) {
const opts = state.opts || {};
const replacementValue = opts.replacementValue || '__dirname';
const {node} = path;
if (
node.callee.type === 'MemberExpression' &&
node.callee.object.type === 'Identifier' &&
node.callee.object.name === 'path' &&
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sofisl can we remove the path portion of this check? I noticed that depending on compiler settings this import name might get mangled to something else.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ignore the above. However, for this to not get mangled I had to do import * as path from 'path'; as opposed to import path from 'path';

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ddelgrosso1, we could remove lines 37 and 38, and still preserve matching (tests still behave the same). Should we do that, that way it's not so fragile?

node.callee.property.type === 'Identifier' &&
node.callee.property.name === 'dirname' &&
node.arguments[0].type === 'CallExpression' &&
node.arguments[0].callee.type === 'Identifier' &&
node.arguments[0].callee.name === 'fileURLToPath' &&
node.arguments[0].arguments[0].type === 'MemberExpression' &&
node.arguments[0].arguments[0].object.type === 'MetaProperty' &&
node.arguments[0].arguments[0].object.meta.type === 'Identifier' &&
node.arguments[0].arguments[0].object.meta.name === 'import' &&
node.arguments[0].arguments[0].object.property.type ===
'Identifier' &&
node.arguments[0].arguments[0].object.property.name === 'meta' &&
node.arguments[0].arguments[0].property.type === 'Identifier' &&
node.arguments[0].arguments[0].property.name === 'url'
) {
const replacement = smart.ast`${replacementValue}` as Statement;
path.replaceWith(replacement);
}
},
},
};
}
44 changes: 44 additions & 0 deletions tools/src/toggleESMFlagVariable.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {Visitor, types} from '@babel/core';

export interface PluginOptions {
opts?: {
variableIdentifier?: string;
replacementValue?: boolean;
};
}

export default function toggleESMFlagVariable(): {
visitor: Visitor<PluginOptions>;
} {
return {
visitor: {
VariableDeclarator(path, state) {
const opts = state.opts || {};
const variableIdentifier = opts.variableIdentifier || 'isEsm';
const replacementValue = opts.replacementValue || false;
const {node} = path;
const identifier = node.id as types.Identifier;
if (
identifier.name === variableIdentifier &&
node.init?.type === 'BooleanLiteral'
) {
node.init.value = replacementValue;
}
},
},
};
}
Loading