feat: add TypeScript samples support (#3029)

* chore: add owlbot * add new lines * Add owlbot.py * Rename .Owlbot.yaml to .OwlBot.yaml * Rename .Owlbot.lock.yaml to .OwlBot.lock.yaml * updates * test typeless bot * cleanup owlbot.py * add typescript support for tests * update copyright year * update copyright year * add ts-node dependency * update ts support * linting * remove comment * add npm build step * change build cmd * test updating pr * update comments * updated owlbot yaml * Update .github/.OwlBot.lock.yaml Co-authored-by: Anthonios Partheniou <[email protected]> * Update .OwlBot.lock.yaml * update owlbot yaml * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * remove extra comment * Update createJob.js * update tsconfig * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add export filter * update filter * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * owlbot update * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * owlbot update * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * final updates * license * update linter * add build step * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * more updates * updates * eslint * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix gts check * reduce ecmascript version * fixes * add express types * skiplibcheck * remove global tsconfig * use root gts rules * add root=true * add extends to individual dirs * update workflow * reorganize * update ts workflow * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * revert ccai change * revert ccai change part deux * fix spacing * appengine include ts only * eslintignore * update package json * exclude test files * fix newline * updates based on feedback * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Anthonios Partheniou <[email protected]> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
GoogleCloudPlatform · Feb 23, 2023 · 8fc8d3a · 8fc8d3a
1 parent ca863d0
commit 8fc8d3a
Show file tree

Hide file tree

Showing 20 changed files with 417 additions and 72 deletions.
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -1,10 +1,12 @@
 {
+  "root": true,
   "extends": "./node_modules/gts/",
   "env": {
     "mocha": true
   },
   "rules": {
     "node/no-missing-require": ["off"], 
-    "node/no-unpublished-require": ["off"]
+    "node/no-unpublished-require": ["off"],
+    "node/no-unsupported-features/es-syntax": ["off"]
   }
-}
+}
diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml
@@ -0,0 +1,17 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+docker:
+  image: gcr.io/cloud-devrel-public-resources/owlbot-nodejs:latest
+  digest: sha256:fe04ae044dadf5ad88d979dbcc85e0e99372fb5d6316790341e6aca5e4e3fbc8
+
diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml
@@ -0,0 +1,17 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+docker:
+  image: gcr.io/cloud-devrel-public-resources/owlbot-nodejs:latest
+
+begin-after-commit-hash: db31e3ff07d737b61ce968625aabbf660501688c
diff --git a/.github/workflows/appengine-typescript.yaml b/.github/workflows/appengine-typescript.yaml
@@ -1,3 +1,17 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: appengine-typescript
 on:
   push:
@@ -26,6 +40,9 @@ jobs:
       contents: 'write'
       pull-requests: 'write'
       id-token: 'write'
+    defaults:
+      run:
+        working-directory: appengine/typescript
     steps:
     - uses: actions/[email protected]
       with:
@@ -40,9 +57,9 @@ jobs:
       with:
         node-version: 16
     - run: npm install
-      working-directory: appengine/typescript
+      working-directory: .
+    - run: npm install
     - run: npm test
-      working-directory: appengine/typescript
       env:
         MOCHA_REPORTER_SUITENAME: appengine_typescript
         MOCHA_REPORTER_OUTPUT: appengine_typescript_sponge_log.xml

diff --git a/.github/workflows/scheduler.yaml b/.github/workflows/scheduler.yaml
@@ -1,3 +1,17 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 name: scheduler
 on:
   push:
@@ -26,6 +40,9 @@ jobs:
       contents: 'write'
       pull-requests: 'write'
       id-token: 'write'
+    defaults:
+      run:
+        working-directory: scheduler
     steps:
     - uses: actions/[email protected]
       with:
@@ -40,9 +57,10 @@ jobs:
       with:
         node-version: 16
     - run: npm install
-      working-directory: scheduler
+      working-directory: .
+    - run: npm install
+    - run: npm run build
     - run: npm test
-      working-directory: scheduler
       env:
         MOCHA_REPORTER_SUITENAME: scheduler
         MOCHA_REPORTER_OUTPUT: scheduler_sponge_log.xml

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -42,6 +42,25 @@ For new samples, a GitHub Actions workflow should be created to run your tests o
 > All tests need a corresponding job file outside of GitHub. If you are a Googler, please provide the CL alongside your PR. See the internal codelab for Kokoro for details. If you don't work at Google, the person reviewing your PR will create the job config for you.
 
 
+### TypeScript Support
+
+This repository also supports TypeScript samples. We use the [typeless-sample-bot](https://github.com/googleapis/google-cloud-node/tree/main/packages/typeless-sample-bot) to convert TypeScript samples to pure JavaScript, which avoids having to maintain both TypeScript and JavaScript variants.
+
+If you choose to write a TypeScript based sample, please follow these guidelines:
+
+* **Testing**: Use a `.ts` test runner, which is executed by the `npm test` command `mocha --require ts-node/register test/*.ts` in `package.json`. **Do not** check in any generated `.js` tests from running `npm build`.
+* **Imports** Use an `import` statement at the beginning of the file to enable importing types. Within each "region tag," import required libraries with `required`. Each of these region tag sections are directly embedded in the Cloud documentation, so the imports help show our users which libraries are needed.
+* **Linting** See the example [.eslintrc.yml](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/main/scheduler/.eslintrc.yml).
+* **JavaScript** Do not modify any `.js` files. The [typeless-sample-bot](https://github.com/googleapis/google-cloud-node/tree/main/packages/typeless-sample-bot) will overwrite them as it converts from TypeScript into JavaScript.
+* **package.json**: See a full set of npm targets in the scheduler [package.json](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/main/scheduler/package.json). Include any relevant `@types` in your `devDependencies` section. 
+* **tsconfig.json**:
+  * Include a [tsconfig.json](https://www.typescriptlang.org/docs/handbook/tsconfig-json.html) in the root of your sample directory.
+  * You can find a minimal example [here](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/main/scheduler/tsconfig.json).
+  * It is recommended to set `noImplicitAny` to `false`, but it may be needed to set this to `true` if you haven't fully migrated the sample to TypeScript.
+  * Add an `excludes` entry with your test files to avoid building `*.js` versions.
+
+You can look at the [scheduler](https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/main/scheduler) sample directory for an example of a TypeScript sample and its matching test runner.
+
 ### Style
 
 The [Google Cloud Samples Style Guide][style-guide] is considered the primary

diff --git a/appengine/typescript/.eslintrc.json b/appengine/typescript/.eslintrc.json
@@ -0,0 +1,6 @@
+{
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "sourceType": "module"
+  }
+}
diff --git a/appengine/typescript/index.ts b/appengine/typescript/index.ts
@@ -12,12 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import express = require('express');
-
+import express, {Express, Request, Response} from 'express';
 const PORT = process.env.PORT || 8080;
-const app = express();
+const app: Express = express();
 
-app.get('/', (req, res) => {
+app.get('/', (req: Request, res: Response) => {
   res.send('🎉 Hello TypeScript! 🎉');
 });
 

diff --git a/appengine/typescript/package.json b/appengine/typescript/package.json
@@ -9,26 +9,27 @@
     "node": "16.x.x"
   },
   "scripts": {
-    "prepare": "npm run gcp-build",
-    "pretest": "npm run gcp-build",
+    "prepare": "npm run compile",
+    "pretest": "npm run compile",
     "test": "mocha test/*.test.js --exit",
-    "posttest": "npm run lint",
-    "lint": "tslint -p .",
+    "lint": "gts lint",
     "start": "node ./index.js",
-    "gcp-build": "tsc -p .",
-    "deploy": "gcloud app deploy"
+    "deploy": "gcloud app deploy",
+    "clean": "gts clean",
+    "compile": "tsc -p .",
+    "fix": "gts fix",
+    "build": "tsc -p ."
   },
   "dependencies": {
-    "express": "^4.16.3",
-    "typescript": "^4.0.0"
+    "express": "^4.16.3"
   },
   "devDependencies": {
-    "mocha": "^10.0.0",
-    "@types/express": "^4.16.0",
-    "@types/node": "^14.14.7",
-    "chai": "^4.2.0",
-    "tslint": "^6.0.0",
-    "typescript": "^4.0.0",
+    "@types/express": "^4.17.17",
+    "@types/node": "^18.13.0",
+    "chai": "^4.3.7",
+    "gts": "^3.1.1",
+    "mocha": "^10.2.0",
+    "typescript": "^4.9.5",
     "wait-port": "^1.0.0"
   }
 }
diff --git a/appengine/typescript/tsconfig.json b/appengine/typescript/tsconfig.json
@@ -1,9 +1,10 @@
 {
   "compilerOptions": {
-    "target": "es2016",
-    "module": "commonjs"
-  },
-  "include": [
-    "*.ts"
-  ]
+    "module": "CommonJS",
+    "target": "ESNext",
+    "strict": true,
+    "noImplicitAny": false,
+    "esModuleInterop": true,
+    "moduleResolution": "node"
+  }
 }
diff --git a/appengine/typescript/tslint.json b/appengine/typescript/tslint.json
diff --git a/owlbot.py b/owlbot.py
@@ -0,0 +1,148 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import subprocess
+from pathlib import Path
+
+from synthtool import shell
+from synthtool.log import logger
+
+_TOOLS_DIRECTORY = "/synthtool"
+_EXCLUDED_DIRS = [r"node_modules", r"^\."]
+_TYPELESS_EXPRESSION = "s/export {};$/\\n/"
+_NPM_CONFIG_CACHE = "/var/tmp/.npm"
+
+
+def walk_through_owlbot_dirs(dir: Path, search_for_changed_files: bool) -> list[str]:
+    """
+    Walks through all sample directories
+    Returns:
+    A list of directories
+    """
+    owlbot_dirs: list[str] = []
+    packages_to_exclude = _EXCLUDED_DIRS
+    if search_for_changed_files:
+        try:
+            # Need to run this step first in the post processor since we only clone
+            # the branch the PR is on in the Docker container
+            output = subprocess.run(
+                ["git", "fetch", "origin", "main:main", "--deepen=200"], check=False
+            )
+            output.check_returncode()
+        except subprocess.CalledProcessError as e:
+            if e.returncode == 128:
+                logger.info(f"Error: ${e.output}; skipping fetching main")
+            else:
+                raise e
+    for path_object in dir.glob("**/package.json"):
+        object_dir = str(Path(path_object).parents[0])
+        if (
+            path_object.is_file()
+            and object_dir != str(dir)
+            and not re.search(
+                "(?:% s)" % "|".join(packages_to_exclude), str(Path(path_object))
+            )
+        ):
+            if search_for_changed_files:
+                if (
+                    subprocess.run(
+                        ["git", "diff", "--quiet", "main...", object_dir], check=False
+                    ).returncode
+                    == 1
+                ):
+                    owlbot_dirs.append(object_dir)
+            else:
+                owlbot_dirs.append(object_dir)
+    for path_object in dir.glob("owl-bot-staging/*"):
+        owlbot_dirs.append(
+            f"{Path(path_object).parents[1]}/packages/{Path(path_object).name}"
+        )
+    return owlbot_dirs
+
+
+def typeless_samples_hermetic(
+    output_path: str, targets: str, hide_output: bool = False
+) -> None:
+    """
+    Converts TypeScript samples in the current Node.js library
+    to JavaScript samples. Run this step before fix() and friends.
+    Assumes that typeless-sample-bot is already installed in a well
+    known location on disk (node_modules/.bin).
+    This is currently an optional, opt-in part of an individual repo's
+    OwlBot.py, and must be called from there before calling owlbot_main.
+    """
+    logger.debug("Run typeless sample bot")
+    shell.run(
+        [
+            f"{_TOOLS_DIRECTORY}/node_modules/.bin/typeless-sample-bot",
+            "--outputpath",
+            output_path,
+            "--targets",
+            targets,
+            "--recursive",
+        ],
+        check=False,
+        hide_output=hide_output,
+    )
+
+
+def trim(targets: str, hide_output: bool = False) -> None:
+    """
+    Fixes the formatting of generated JS files
+    """
+    logger.debug("Trim generated files")
+    for file in os.listdir(f"{targets}"):
+        if file.endswith(".js"):
+            logger.debug(f"Updating {targets}/{file}")
+            shell.run(
+                ["sed", "-i", "-e", _TYPELESS_EXPRESSION, f"{targets}/{file}"],
+                check=False,
+                hide_output=hide_output,
+            )
+
+
+def fix_hermetic(targets=".", hide_output=False):
+    """
+    Fixes the formatting in the current Node.js library. It assumes that gts
+    is already installed in a well known location on disk (node_modules/.bin).
+    """
+    logger.debug("Copy eslint config")
+    shell.run(
+        ["cp", "-r", f"{_TOOLS_DIRECTORY}/node_modules", "."],
+        check=True,
+        hide_output=hide_output,
+    )
+    logger.debug("Running fix...")
+    shell.run(
+        [f"{_TOOLS_DIRECTORY}/node_modules/.bin/gts", "fix", f"{targets}"],
+        check=False,
+        hide_output=hide_output,
+    )
+
+
+# Avoid "Your cache folder contains root-owned files" error
+os.environ["npm_config_cache"] = _NPM_CONFIG_CACHE
+
+# Retrieve list of directories
+dirs: list[str] = walk_through_owlbot_dirs(Path.cwd(), search_for_changed_files=True)
+for d in dirs:
+    logger.debug(f"Directory: {d}")
+    # Run typeless bot to convert from TS -> JS
+    typeless_samples_hermetic(output_path=d, targets=d)
+    # Remove extra characters
+    trim(targets=d)
+    # Fix formatting
+    fix_hermetic(targets=d)