From e8f5b0970e830f9f5b6681da260e9b04d3aca39f Mon Sep 17 00:00:00 2001 From: Kevin Stubbs Date: Mon, 4 Mar 2024 14:35:34 +0300 Subject: [PATCH 1/4] Refactor drupal & markdown importers to separate files. Implement smart component replacement for drupal import. --- packages/cli/package.json | 1 + .../commands/{import.ts => import/drupal.ts} | 158 ++++++------------ packages/cli/src/cli/commands/import/index.ts | 4 + .../cli/src/cli/commands/import/markdown.ts | 117 +++++++++++++ packages/cli/src/lib/addonApiHelper.ts | 35 ++++ pnpm-lock.yaml | 17 +- 6 files changed, 220 insertions(+), 112 deletions(-) rename packages/cli/src/cli/commands/{import.ts => import/drupal.ts} (62%) create mode 100644 packages/cli/src/cli/commands/import/index.ts create mode 100644 packages/cli/src/cli/commands/import/markdown.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 50c18aaa..fb646b0a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -49,6 +49,7 @@ "google-auth-library": "^9.4.0", "googleapis": "^129.0.0", "inquirer": "^8.2.6", + "node-html-parser": "^6.1.12", "nunjucks": "^3.2.4", "octokit": "^3.1.2", "open": "^9.1.0", diff --git a/packages/cli/src/cli/commands/import.ts b/packages/cli/src/cli/commands/import/drupal.ts similarity index 62% rename from packages/cli/src/cli/commands/import.ts rename to packages/cli/src/cli/commands/import/drupal.ts index 980f7d75..0c310c41 100644 --- a/packages/cli/src/cli/commands/import.ts +++ b/packages/cli/src/cli/commands/import/drupal.ts @@ -1,22 +1,17 @@ import { randomUUID } from "crypto"; -import * as fs from "fs"; import { exit } from "process"; import axios, { AxiosError } from "axios"; import Promise from "bluebird"; import chalk from "chalk"; -import { parseFromString } from "dom-parser"; import type { GaxiosResponse } from "gaxios"; import { OAuth2Client } from "google-auth-library"; import { drive_v3, google } from "googleapis"; -import ora from "ora"; +import { HTMLElement, parse } from "node-html-parser"; import queryString from "query-string"; -import showdown from "showdown"; -import AddOnApiHelper from "../../lib/addonApiHelper"; -import { getLocalAuthDetails } from "../../lib/localStorage"; -import { Logger } from "../../lib/logger"; -import { errorHandler } from "../exceptions"; - -const HEADING_TAGS = ["h1", "h2", "h3", "title"]; +import { getLocalAuthDetails } from "../../../lib/localStorage"; +import { Logger } from "../../../lib/logger"; +import { errorHandler } from "../../exceptions"; +import AddOnApiHelper from "../../../lib/addonApiHelper"; type DrupalImportParams = { baseUrl: string; @@ -182,7 +177,9 @@ export const importFromDrupal = errorHandler( (x) => x.id === post.relationships.field_author.data.id, )?.attributes?.title; - const res = (await drive.files.create({ + // Initially create a blank document, just to get an article id + // that we can work with for further steps, such as adding smart components. + let res = (await drive.files.create({ requestBody: { // Name from the article. name: post.attributes.title, @@ -191,7 +188,7 @@ export const importFromDrupal = errorHandler( }, media: { mimeType: "text/html", - body: post.attributes.body.processed, + body: "", }, })) as GaxiosResponse; const fileId = res.data.id; @@ -203,6 +200,21 @@ export const importFromDrupal = errorHandler( // Add it to the PCC site. await AddOnApiHelper.getDocument(fileId, true); + // Set the document's content. + res = (await drive.files.update({ + requestBody: { + id: fileId, + mimeType: "application/vnd.google-apps.document", + }, + media: { + mimeType: "text/html", + body: await processHTMLForSmartComponents( + post.attributes.body.processed, + fileId, + ), + }, + })) as GaxiosResponse; + try { await AddOnApiHelper.updateDocument( fileId, @@ -241,104 +253,38 @@ export const importFromDrupal = errorHandler( }, ); -type MarkdownImportParams = { - filePath: string; - siteId: string; - verbose: boolean; - publish: boolean; -}; - -export const importFromMarkdown = errorHandler( - async ({ filePath, siteId, verbose, publish }: MarkdownImportParams) => { - const logger = new Logger(); +async function processHTMLForSmartComponents(html: string, articleId: string) { + const root = parse(html); + const iframeNodes: HTMLElement[] = + (root.querySelector("iframe")?.childNodes as HTMLElement[]) ?? []; - if (!fs.existsSync(filePath)) { - logger.error( - chalk.red( - `ERROR: Could not find markdown file at given path (${filePath})`, - ), - ); - exit(1); - } + await Promise.all( + iframeNodes.map(async (node) => { + let src = node.getAttribute("src"); - // Prepare article content and title - const content = fs.readFileSync(filePath).toString(); - - // Check user has required permission to create drive file - await AddOnApiHelper.getIdToken([ - "https://www.googleapis.com/auth/drive.file", - ]); - const authDetails = await getLocalAuthDetails(); - if (!authDetails) { - logger.error(chalk.red(`ERROR: Failed to retrieve login details.`)); - exit(1); - } + if (src == null) return; - // Create Google Doc - const spinner = ora("Creating document on the Google Drive...").start(); - const oauth2Client = new OAuth2Client(); - oauth2Client.setCredentials(authDetails); - const drive = google.drive({ - version: "v3", - auth: oauth2Client, - }); - const converter = new showdown.Converter(); - const html = converter.makeHtml(content); - const dom = parseFromString(html); - - // Derive document's title - let title: string | undefined = undefined; - for (const item of HEADING_TAGS) { - const element = dom.getElementsByTagName(item)[0]; - if (element) { - title = element.textContent; - break; + if (src.includes("oembed?url=")) { + src = decodeURIComponent(src.split("oembed?url=")[1]); } - } - title = title || "Untitled Document"; - - const res = (await drive.files.create({ - requestBody: { - name: title, - mimeType: "application/vnd.google-apps.document", - }, - media: { - mimeType: "text/html", - body: html, - }, - })) as GaxiosResponse; - const fileId = res.data.id; - const fileUrl = `https://docs.google.com/document/d/${fileId}`; - if (!fileId) { - spinner.fail("Failed to create document on the Google Drive."); - exit(1); - } + const componentType = "MEDIA_PREVIEW"; + const componentId = await AddOnApiHelper.createSmartComponent( + articleId, + { + url: src, + canUsePlainIframe: true, + }, + componentType, + ); - // Create PCC document - await AddOnApiHelper.getDocument(fileId, true, title); - // Cannot set metadataFields(title,slug) in the same request since we reset metadataFields - // when changing the siteId. - await AddOnApiHelper.updateDocument( - fileId, - siteId, - title, - [], - null, - verbose, - ); - await AddOnApiHelper.getDocument(fileId, false, title); + node.replaceWith( + parse( + `MEDIA_PREVIEW: ${src}`, + ), + ); + }), + ); - // Publish PCC document - if (publish) { - const { token } = await oauth2Client.getAccessToken(); - await AddOnApiHelper.publishDocument(fileId, token as string); - } - spinner.succeed( - `Successfully created document at below path${ - publish ? " and published it on the PCC." : ":" - }`, - ); - logger.log(chalk.green(fileUrl, "\n")); - }, -); + return root.toString(); +} diff --git a/packages/cli/src/cli/commands/import/index.ts b/packages/cli/src/cli/commands/import/index.ts new file mode 100644 index 00000000..48ecd02d --- /dev/null +++ b/packages/cli/src/cli/commands/import/index.ts @@ -0,0 +1,4 @@ +import { importFromDrupal } from "./drupal"; +import { importFromMarkdown } from "./markdown"; + +export { importFromDrupal, importFromMarkdown }; diff --git a/packages/cli/src/cli/commands/import/markdown.ts b/packages/cli/src/cli/commands/import/markdown.ts new file mode 100644 index 00000000..0e5ba536 --- /dev/null +++ b/packages/cli/src/cli/commands/import/markdown.ts @@ -0,0 +1,117 @@ +import * as fs from "fs"; +import { exit } from "process"; +import chalk from "chalk"; +import { parseFromString } from "dom-parser"; +import type { GaxiosResponse } from "gaxios"; +import { OAuth2Client } from "google-auth-library"; +import { drive_v3, google } from "googleapis"; +import ora from "ora"; +import showdown from "showdown"; +import AddOnApiHelper from "../../../lib/addonApiHelper"; +import { getLocalAuthDetails } from "../../../lib/localStorage"; +import { Logger } from "../../../lib/logger"; +import { errorHandler } from "../../exceptions"; + +const HEADING_TAGS = ["h1", "h2", "h3", "title"]; + +type MarkdownImportParams = { + filePath: string; + siteId: string; + verbose: boolean; + publish: boolean; +}; + +export const importFromMarkdown = errorHandler( + async ({ filePath, siteId, verbose, publish }: MarkdownImportParams) => { + const logger = new Logger(); + + if (!fs.existsSync(filePath)) { + logger.error( + chalk.red( + `ERROR: Could not find markdown file at given path (${filePath})`, + ), + ); + exit(1); + } + + // Prepare article content and title + const content = fs.readFileSync(filePath).toString(); + + // Check user has required permission to create drive file + await AddOnApiHelper.getIdToken([ + "https://www.googleapis.com/auth/drive.file", + ]); + const authDetails = await getLocalAuthDetails(); + if (!authDetails) { + logger.error(chalk.red(`ERROR: Failed to retrieve login details.`)); + exit(1); + } + + // Create Google Doc + const spinner = ora("Creating document on the Google Drive...").start(); + const oauth2Client = new OAuth2Client(); + oauth2Client.setCredentials(authDetails); + const drive = google.drive({ + version: "v3", + auth: oauth2Client, + }); + const converter = new showdown.Converter(); + const html = converter.makeHtml(content); + const dom = parseFromString(html); + + // Derive document's title + let title: string | undefined = undefined; + for (const item of HEADING_TAGS) { + const element = dom.getElementsByTagName(item)[0]; + if (element) { + title = element.textContent; + break; + } + } + title = title || "Untitled Document"; + + const res = (await drive.files.create({ + requestBody: { + name: title, + mimeType: "application/vnd.google-apps.document", + }, + media: { + mimeType: "text/html", + body: html, + }, + })) as GaxiosResponse; + const fileId = res.data.id; + const fileUrl = `https://docs.google.com/document/d/${fileId}`; + + if (!fileId) { + spinner.fail("Failed to create document on the Google Drive."); + exit(1); + } + + // Create PCC document + await AddOnApiHelper.getDocument(fileId, true, title); + // Cannot set metadataFields(title,slug) in the same request since we reset metadataFields + // when changing the siteId. + await AddOnApiHelper.updateDocument( + fileId, + siteId, + title, + [], + null, + verbose, + ); + await AddOnApiHelper.getDocument(fileId, false, title); + + // Publish PCC document + if (publish) { + const { token } = await oauth2Client.getAccessToken(); + await AddOnApiHelper.publishDocument(fileId, token as string); + } + spinner.succeed( + `Successfully created document at below path${ + publish ? " and published it on the PCC." : ":" + }`, + ); + logger.log(chalk.green(fileUrl, "\n")); + }, +); diff --git a/packages/cli/src/lib/addonApiHelper.ts b/packages/cli/src/lib/addonApiHelper.ts index 00a069cc..0aa92d71 100644 --- a/packages/cli/src/lib/addonApiHelper.ts +++ b/packages/cli/src/lib/addonApiHelper.ts @@ -1,3 +1,4 @@ +import { SmartComponentMap } from "@pantheon-systems/pcc-sdk-core/types"; import axios, { AxiosError, HttpStatusCode } from "axios"; import { Credentials } from "google-auth-library"; import ora from "ora"; @@ -226,6 +227,40 @@ class AddOnApiHelper { } } + static async createSmartComponent( + articleId: string, + attributes: { [key: string]: string | number | boolean | null | undefined }, + componentType: string, + ): Promise { + const idToken = await this.getIdToken(); + + try { + return ( + await axios.post( + `${API_KEY_ENDPOINT}/components`, + { + articleId, + attributes, + componentType, + }, + { + headers: { + Authorization: `Bearer ${idToken}`, + }, + }, + ) + ).data.id; + } catch (err) { + if ( + (err as { response: { status: number } }).response.status === + HttpStatusCode.NotFound + ) + throw new HTTPNotFound(); + + throw err; + } + } + static async createSite(url: string): Promise { const idToken = await this.getIdToken(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ddbd13e0..13700a35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -115,7 +115,7 @@ importers: packages/browser: dependencies: '@pantheon-systems/pcc-sdk-core': - specifier: ^3.1.1 + specifier: ^3.1.2 version: link:../core devDependencies: eslint: @@ -169,6 +169,9 @@ importers: inquirer: specifier: ^8.2.6 version: 8.2.6 + node-html-parser: + specifier: ^6.1.12 + version: 6.1.12 nunjucks: specifier: ^3.2.4 version: 3.2.4 @@ -12961,7 +12964,6 @@ packages: domhandler: 5.0.3 domutils: 3.1.0 nth-check: 2.1.1 - dev: true /css-selector-parser@1.4.1: resolution: {integrity: sha512-HYPSb7y/Z7BNDCOrakL4raGO2zltZkbeXyAd6Tg9obzix6QhzxCotdBl6VT0Dv4vZfJGVz3WL/xaEI9Ly3ul0g==} @@ -13601,7 +13603,6 @@ packages: domelementtype: 2.3.0 domhandler: 5.0.3 entities: 4.5.0 - dev: true /domain-browser@4.23.0: resolution: {integrity: sha512-ArzcM/II1wCCujdCNyQjXrAFwS4mrLh4C7DZWlaI8mdh7h3BfKdNd3bKXITfl2PT9FtfQqaGvhi1vPRQPimjGA==} @@ -13630,7 +13631,6 @@ packages: engines: {node: '>= 4'} dependencies: domelementtype: 2.3.0 - dev: true /domutils@2.8.0: resolution: {integrity: sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==} @@ -13645,7 +13645,6 @@ packages: dom-serializer: 2.0.0 domelementtype: 2.3.0 domhandler: 5.0.3 - dev: true /dot-case@3.0.4: resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} @@ -13853,7 +13852,6 @@ packages: /entities@4.5.0: resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} engines: {node: '>=0.12'} - dev: true /env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} @@ -20711,6 +20709,13 @@ packages: he: 1.2.0 dev: false + /node-html-parser@6.1.12: + resolution: {integrity: sha512-/bT/Ncmv+fbMGX96XG9g05vFt43m/+SYKIs9oAemQVYyVcZmDAI2Xq/SbNcpOA35eF0Zk2av3Ksf+Xk8Vt8abA==} + dependencies: + css-select: 5.1.0 + he: 1.2.0 + dev: false + /node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} From 7f135914c1dc3b5cb5d088508b6b79530b16d951 Mon Sep 17 00:00:00 2001 From: Kevin Stubbs Date: Mon, 5 Aug 2024 16:24:16 +0300 Subject: [PATCH 2/4] Combine duplicate code related to creating a file. --- .../cli/src/cli/commands/import/common.ts | 55 +++++++++++++++++++ .../cli/src/cli/commands/import/drupal.ts | 23 +++----- .../cli/src/cli/commands/import/markdown.ts | 43 ++------------- 3 files changed, 69 insertions(+), 52 deletions(-) create mode 100644 packages/cli/src/cli/commands/import/common.ts diff --git a/packages/cli/src/cli/commands/import/common.ts b/packages/cli/src/cli/commands/import/common.ts new file mode 100644 index 00000000..f7e2c129 --- /dev/null +++ b/packages/cli/src/cli/commands/import/common.ts @@ -0,0 +1,55 @@ +import { exit } from "process"; +import chalk from "chalk"; +import type { GaxiosResponse } from "gaxios"; +import { OAuth2Client } from "google-auth-library"; +import { drive_v3, google } from "googleapis"; +import ora from "ora"; +import AddOnApiHelper from "../../../lib/addonApiHelper"; +import { getLocalAuthDetails } from "../../../lib/localStorage"; +import { Logger } from "../../../lib/logger"; + +export async function createFileOnDrive( + requestBody: Partial, + body: string, +) { + const logger = new Logger(); + + // Check user has required permission to create drive file + await AddOnApiHelper.getIdToken([ + "https://www.googleapis.com/auth/drive.file", + ]); + const authDetails = await getLocalAuthDetails(); + if (!authDetails) { + logger.error(chalk.red(`ERROR: Failed to retrieve login details.`)); + exit(1); + } + + // Create Google Doc + const spinner = ora("Creating document on the Google Drive...").start(); + const oauth2Client = new OAuth2Client(); + oauth2Client.setCredentials(authDetails); + const drive = google.drive({ + version: "v3", + auth: oauth2Client, + }); + + const res = (await drive.files.create({ + requestBody: { + ...requestBody, + mimeType: "application/vnd.google-apps.document", + }, + media: { + mimeType: "text/html", + body, + }, + })) as GaxiosResponse; + const fileId = res.data.id; + const fileUrl = `https://docs.google.com/document/d/${fileId}`; + + if (!fileId) { + spinner.fail("Failed to create document on the Google Drive."); + exit(1); + } + + return { fileId, fileUrl, drive, spinner }; +} diff --git a/packages/cli/src/cli/commands/import/drupal.ts b/packages/cli/src/cli/commands/import/drupal.ts index 1dfc8cd6..7e2742fe 100644 --- a/packages/cli/src/cli/commands/import/drupal.ts +++ b/packages/cli/src/cli/commands/import/drupal.ts @@ -12,6 +12,7 @@ import AddOnApiHelper from "../../../lib/addonApiHelper"; import { getLocalAuthDetails } from "../../../lib/localStorage"; import { Logger } from "../../../lib/logger"; import { errorHandler } from "../../exceptions"; +import { createFileOnDrive } from "./common"; type DrupalImportParams = { baseUrl: string; @@ -187,29 +188,21 @@ export const importFromDrupal = errorHandler( // Initially create a blank document, just to get an article id // that we can work with for further steps, such as adding smart components. - let res = (await drive.files.create({ - requestBody: { - // Name from the article. + const { fileId, spinner, drive } = await createFileOnDrive( + { name: post.attributes.title, - mimeType: "application/vnd.google-apps.document", + parents: [folderId], }, - media: { - mimeType: "text/html", - body: "", - }, - })) as GaxiosResponse; - const fileId = res.data.id; - - if (!fileId) { - throw new Error(`Failed to create file for ${post.attributes.title}`); - } + "", + ); + spinner.succeed(); // Add it to the PCC site. await AddOnApiHelper.getDocument(fileId, true); // Set the document's content. - res = (await drive.files.update({ + const res = (await drive.files.update({ requestBody: { id: fileId, mimeType: "application/vnd.google-apps.document", diff --git a/packages/cli/src/cli/commands/import/markdown.ts b/packages/cli/src/cli/commands/import/markdown.ts index 0e5ba536..1c8c498c 100644 --- a/packages/cli/src/cli/commands/import/markdown.ts +++ b/packages/cli/src/cli/commands/import/markdown.ts @@ -11,6 +11,7 @@ import AddOnApiHelper from "../../../lib/addonApiHelper"; import { getLocalAuthDetails } from "../../../lib/localStorage"; import { Logger } from "../../../lib/logger"; import { errorHandler } from "../../exceptions"; +import { createFileOnDrive } from "./common"; const HEADING_TAGS = ["h1", "h2", "h3", "title"]; @@ -37,24 +38,6 @@ export const importFromMarkdown = errorHandler( // Prepare article content and title const content = fs.readFileSync(filePath).toString(); - // Check user has required permission to create drive file - await AddOnApiHelper.getIdToken([ - "https://www.googleapis.com/auth/drive.file", - ]); - const authDetails = await getLocalAuthDetails(); - if (!authDetails) { - logger.error(chalk.red(`ERROR: Failed to retrieve login details.`)); - exit(1); - } - - // Create Google Doc - const spinner = ora("Creating document on the Google Drive...").start(); - const oauth2Client = new OAuth2Client(); - oauth2Client.setCredentials(authDetails); - const drive = google.drive({ - version: "v3", - auth: oauth2Client, - }); const converter = new showdown.Converter(); const html = converter.makeHtml(content); const dom = parseFromString(html); @@ -70,23 +53,10 @@ export const importFromMarkdown = errorHandler( } title = title || "Untitled Document"; - const res = (await drive.files.create({ - requestBody: { - name: title, - mimeType: "application/vnd.google-apps.document", - }, - media: { - mimeType: "text/html", - body: html, - }, - })) as GaxiosResponse; - const fileId = res.data.id; - const fileUrl = `https://docs.google.com/document/d/${fileId}`; - - if (!fileId) { - spinner.fail("Failed to create document on the Google Drive."); - exit(1); - } + const { fileId, fileUrl, spinner } = await createFileOnDrive( + { name: title }, + html, + ); // Create PCC document await AddOnApiHelper.getDocument(fileId, true, title); @@ -104,8 +74,7 @@ export const importFromMarkdown = errorHandler( // Publish PCC document if (publish) { - const { token } = await oauth2Client.getAccessToken(); - await AddOnApiHelper.publishDocument(fileId, token as string); + await AddOnApiHelper.publishDocument(fileId); } spinner.succeed( `Successfully created document at below path${ From 436179d3fd1d9b481cb29b613bf07a215bb7ced5 Mon Sep 17 00:00:00 2001 From: Kevin Stubbs Date: Mon, 5 Aug 2024 16:25:11 +0300 Subject: [PATCH 3/4] Update changeset --- .changeset/chatty-singers-scream.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/chatty-singers-scream.md diff --git a/.changeset/chatty-singers-scream.md b/.changeset/chatty-singers-scream.md new file mode 100644 index 00000000..16d820e4 --- /dev/null +++ b/.changeset/chatty-singers-scream.md @@ -0,0 +1,5 @@ +--- +"@pantheon-systems/pcc-cli": minor +--- + +Internal refactoring of 3rd-party file imports. From 68da81226427e28d91dd2ea7b97b10ffaca07ba1 Mon Sep 17 00:00:00 2001 From: Kevin Stubbs Date: Tue, 6 Aug 2024 17:53:39 +0300 Subject: [PATCH 4/4] Added --publish flag to Drupal import. Fixed drupal import bugs. Added link to folder after import completes. Fixed createSmartComponent function call using incorrect endpoint. --- .changeset/afraid-maps-smell.md | 7 ++ .../cli/src/cli/commands/import/common.ts | 45 ++++++---- .../cli/src/cli/commands/import/drupal.ts | 34 +++++--- packages/cli/src/cli/index.ts | 7 ++ packages/cli/src/lib/addonApiHelper.ts | 86 ++++++++++--------- 5 files changed, 109 insertions(+), 70 deletions(-) create mode 100644 .changeset/afraid-maps-smell.md diff --git a/.changeset/afraid-maps-smell.md b/.changeset/afraid-maps-smell.md new file mode 100644 index 00000000..4935f0fd --- /dev/null +++ b/.changeset/afraid-maps-smell.md @@ -0,0 +1,7 @@ +--- +"@pantheon-systems/pcc-cli": patch +--- + +Added --publish flag to Drupal import. Fixed drupal import bugs. Added link to +folder after import completes. Fixed createSmartComponent function call using +incorrect endpoint. diff --git a/packages/cli/src/cli/commands/import/common.ts b/packages/cli/src/cli/commands/import/common.ts index f7e2c129..a3b2f1d2 100644 --- a/packages/cli/src/cli/commands/import/common.ts +++ b/packages/cli/src/cli/commands/import/common.ts @@ -8,31 +8,38 @@ import AddOnApiHelper from "../../../lib/addonApiHelper"; import { getLocalAuthDetails } from "../../../lib/localStorage"; import { Logger } from "../../../lib/logger"; -export async function createFileOnDrive( - requestBody: Partial, - body: string, -) { +export async function createFileOnDrive({ + requestBody, + body, + drive, +}: { + requestBody: Partial; + body: string; + drive?: drive_v3.Drive; +}) { const logger = new Logger(); - // Check user has required permission to create drive file - await AddOnApiHelper.getIdToken([ - "https://www.googleapis.com/auth/drive.file", - ]); - const authDetails = await getLocalAuthDetails(); - if (!authDetails) { - logger.error(chalk.red(`ERROR: Failed to retrieve login details.`)); - exit(1); + if (!drive) { + // Check user has required permission to create drive file + await AddOnApiHelper.getIdToken([ + "https://www.googleapis.com/auth/drive.file", + ]); + const authDetails = await getLocalAuthDetails(); + if (!authDetails) { + logger.error(chalk.red(`ERROR: Failed to retrieve login details.`)); + exit(1); + } + + const oauth2Client = new OAuth2Client(); + oauth2Client.setCredentials(authDetails); + drive = google.drive({ + version: "v3", + auth: oauth2Client, + }); } // Create Google Doc const spinner = ora("Creating document on the Google Drive...").start(); - const oauth2Client = new OAuth2Client(); - oauth2Client.setCredentials(authDetails); - const drive = google.drive({ - version: "v3", - auth: oauth2Client, - }); - const res = (await drive.files.create({ requestBody: { ...requestBody, diff --git a/packages/cli/src/cli/commands/import/drupal.ts b/packages/cli/src/cli/commands/import/drupal.ts index 7e2742fe..3c259879 100644 --- a/packages/cli/src/cli/commands/import/drupal.ts +++ b/packages/cli/src/cli/commands/import/drupal.ts @@ -18,6 +18,7 @@ type DrupalImportParams = { baseUrl: string; siteId: string; verbose: boolean; + automaticallyPublish: boolean; }; interface DrupalPost { @@ -76,7 +77,12 @@ async function getDrupalPosts(url: string) { } export const importFromDrupal = errorHandler( - async ({ baseUrl, siteId, verbose }: DrupalImportParams) => { + async ({ + baseUrl, + siteId, + verbose, + automaticallyPublish, + }: DrupalImportParams) => { const logger = new Logger(); if (baseUrl) { @@ -100,7 +106,7 @@ export const importFromDrupal = errorHandler( } } - await AddOnApiHelper.getIdToken([ + const idToken = await AddOnApiHelper.getIdToken([ "https://www.googleapis.com/auth/drive.file", ]); @@ -118,11 +124,12 @@ export const importFromDrupal = errorHandler( auth: oauth2Client, }); + const folderName = `PCC Import from Drupal on ${new Date().toLocaleDateString()} unique id: ${randomUUID()}`; const folderRes = (await drive.files .create({ fields: "id,name", requestBody: { - name: `PCC Import from Drupal on ${new Date().toLocaleDateString()} unique id: ${randomUUID()}`, + name: folderName, mimeType: "application/vnd.google-apps.folder", }, }) @@ -188,23 +195,24 @@ export const importFromDrupal = errorHandler( // Initially create a blank document, just to get an article id // that we can work with for further steps, such as adding smart components. - const { fileId, spinner, drive } = await createFileOnDrive( - { + const { fileId, spinner } = await createFileOnDrive({ + requestBody: { name: post.attributes.title, parents: [folderId], }, - "", - ); + body: "", + drive, + }); spinner.succeed(); // Add it to the PCC site. - await AddOnApiHelper.getDocument(fileId, true); + await AddOnApiHelper.getDocument(fileId, true, undefined, idToken); // Set the document's content. - const res = (await drive.files.update({ + (await drive.files.update({ + fileId, requestBody: { - id: fileId, mimeType: "application/vnd.google-apps.document", }, media: { @@ -235,7 +243,9 @@ export const importFromDrupal = errorHandler( verbose, ); - await AddOnApiHelper.publishDocument(fileId); + if (automaticallyPublish) { + await AddOnApiHelper.publishDocument(fileId); + } } catch (e) { console.error(e instanceof AxiosError ? e.response?.data : e); throw e; @@ -248,7 +258,7 @@ export const importFromDrupal = errorHandler( logger.log( chalk.green( - `Successfully imported ${allPosts.length} documents into ${folderRes.data.name}`, + `Successfully imported ${allPosts.length} documents into ${folderName} (https://drive.google.com/drive/u/0/folders/${folderRes.data.id})`, ), ); }, diff --git a/packages/cli/src/cli/index.ts b/packages/cli/src/cli/index.ts index f506a672..553e2b77 100755 --- a/packages/cli/src/cli/index.ts +++ b/packages/cli/src/cli/index.ts @@ -666,6 +666,12 @@ yargs(hideBin(process.argv)) default: false, demandOption: false, }) + .option("publish", { + describe: "Automatically publish each imported article.", + type: "boolean", + default: false, + demandOption: false, + }) .demandOption(["baseUrl", "siteId"]); }, async (args) => @@ -673,6 +679,7 @@ yargs(hideBin(process.argv)) baseUrl: args.baseUrl as string, siteId: args.siteId as string, verbose: args.verbose as boolean, + automaticallyPublish: args.publish as boolean, }), ) .command( diff --git a/packages/cli/src/lib/addonApiHelper.ts b/packages/cli/src/lib/addonApiHelper.ts index 0bed4260..7bfa3c35 100644 --- a/packages/cli/src/lib/addonApiHelper.ts +++ b/packages/cli/src/lib/addonApiHelper.ts @@ -9,6 +9,11 @@ import { getApiConfig } from "./apiConfig"; import { getLocalAuthDetails } from "./localStorage"; import { toKebabCase } from "./utils"; +export interface AuthToken { + idToken: string | null | undefined; + oauthToken: string | null | undefined; +} + class AddOnApiHelper { static async getToken(code: string): Promise { const resp = await axios.post( @@ -41,11 +46,7 @@ class AddOnApiHelper { } } - static async getIdToken( - requiredScopes?: string[], - withAuthToken?: true, - ): Promise<{ idToken: string; oauthToken: string }>; - static async getIdToken(requiredScopes?: string[], withAuthToken?: boolean) { + static async getIdToken(requiredScopes?: string[]): Promise { let authDetails = await getLocalAuthDetails(requiredScopes); // If auth details not found, try user logging in @@ -57,17 +58,21 @@ class AddOnApiHelper { if (!authDetails) throw new UserNotLoggedIn(); } - return withAuthToken - ? { idToken: authDetails.id_token, oauthToken: authDetails.access_token } - : authDetails.id_token; + return { + idToken: authDetails.id_token, + oauthToken: authDetails.access_token, + }; } static async getDocument( documentId: string, insertIfMissing = false, title?: string, + authToken?: AuthToken, ): Promise
{ - const idToken = await this.getIdToken(); + const { idToken, oauthToken } = + authToken || + (await await this.getIdToken(["https://www.googleapis.com/auth/drive"])); const resp = await axios.get( `${(await getApiConfig()).DOCUMENT_ENDPOINT}/${documentId}`, @@ -80,6 +85,8 @@ class AddOnApiHelper { }, headers: { Authorization: `Bearer ${idToken}`, + "Content-Type": "application/json", + "oauth-token": oauthToken, }, }, ); @@ -92,7 +99,9 @@ class AddOnApiHelper { fieldTitle: string, fieldType: string, ): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await await this.getIdToken([ + "https://www.googleapis.com/auth/drive", + ]); await axios.post( `${(await getApiConfig()).SITE_ENDPOINT}/${siteId}/metadata`, @@ -123,7 +132,7 @@ class AddOnApiHelper { verbose?: boolean, ): Promise
{ - const idToken = await this.getIdToken(); + const { idToken, oauthToken } = await this.getIdToken(); if (verbose) { console.log("update document", { @@ -149,6 +158,7 @@ class AddOnApiHelper { headers: { Authorization: `Bearer ${idToken}`, "Content-Type": "application/json", + "oauth-token": oauthToken, }, }, ); @@ -157,10 +167,9 @@ class AddOnApiHelper { } static async publishDocument(documentId: string) { - const { idToken, oauthToken } = await this.getIdToken( - ["https://www.googleapis.com/auth/drive"], - true, - ); + const { idToken, oauthToken } = await this.getIdToken([ + "https://www.googleapis.com/auth/drive", + ]); if (!idToken || !oauthToken) { throw new UserNotLoggedIn(); @@ -168,7 +177,7 @@ class AddOnApiHelper { const resp = await axios.post<{ url: string }>( `${(await getApiConfig()).DOCUMENT_ENDPOINT}/${documentId}/publish`, - null, + {}, { headers: { Authorization: `Bearer ${idToken}`, @@ -199,10 +208,9 @@ class AddOnApiHelper { baseUrl?: string; }, ): Promise { - const { idToken, oauthToken } = await this.getIdToken( - ["https://www.googleapis.com/auth/drive"], - true, - ); + const { idToken, oauthToken } = await this.getIdToken([ + "https://www.googleapis.com/auth/drive", + ]); if (!idToken || !oauthToken) { throw new UserNotLoggedIn(); @@ -230,7 +238,7 @@ class AddOnApiHelper { static async createApiKey({ siteId, }: { siteId?: string } = {}): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.post( (await getApiConfig()).API_KEY_ENDPOINT, @@ -247,7 +255,7 @@ class AddOnApiHelper { } static async listApiKeys(): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.get((await getApiConfig()).API_KEY_ENDPOINT, { headers: { @@ -259,7 +267,7 @@ class AddOnApiHelper { } static async revokeApiKey(id: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); try { await axios.delete(`${(await getApiConfig()).API_KEY_ENDPOINT}/${id}`, { @@ -281,12 +289,12 @@ class AddOnApiHelper { attributes: { [key: string]: string | number | boolean | null | undefined }, componentType: string, ): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); try { return ( await axios.post( - `${API_KEY_ENDPOINT}/components`, + `${(await getApiConfig()).API_KEY_ENDPOINT}/components`, { articleId, attributes, @@ -311,7 +319,7 @@ class AddOnApiHelper { } static async createSite(url: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.post( (await getApiConfig()).SITE_ENDPOINT, @@ -330,7 +338,7 @@ class AddOnApiHelper { transferToSiteId: string | null | undefined, force: boolean, ): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.delete( queryString.stringifyUrl({ @@ -354,7 +362,7 @@ class AddOnApiHelper { }: { withConnectionStatus?: boolean; }): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.get((await getApiConfig()).SITE_ENDPOINT, { headers: { @@ -369,7 +377,7 @@ class AddOnApiHelper { } static async getSite(siteId: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.get( `${(await getApiConfig()).SITE_ENDPOINT}/${siteId}`, @@ -384,7 +392,7 @@ class AddOnApiHelper { } static async updateSite(id: string, url: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); await axios.patch( `${(await getApiConfig()).SITE_ENDPOINT}/${id}`, @@ -398,7 +406,7 @@ class AddOnApiHelper { } static async getServersideComponentSchema(id: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); await axios.get( `${(await getApiConfig()).SITE_ENDPOINT}/${id}/components`, @@ -414,7 +422,7 @@ class AddOnApiHelper { id: string, componentSchema: typeof SmartComponentMapZod, ): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); await axios.post( `${(await getApiConfig()).SITE_ENDPOINT}/${id}/components`, @@ -430,7 +438,7 @@ class AddOnApiHelper { } static async removeComponentSchema(id: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); await axios.delete( `${(await getApiConfig()).SITE_ENDPOINT}/${id}/components`, @@ -443,7 +451,7 @@ class AddOnApiHelper { } static async listAdmins(id: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); return ( await axios.get(`${(await getApiConfig()).SITE_ENDPOINT}/${id}/admins`, { @@ -455,7 +463,7 @@ class AddOnApiHelper { } static async addAdmin(id: string, email: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); await axios.patch( `${(await getApiConfig()).SITE_ENDPOINT}/${id}/admins`, @@ -471,7 +479,7 @@ class AddOnApiHelper { } static async removeAdmin(id: string, email: string): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); await axios.delete(`${(await getApiConfig()).SITE_ENDPOINT}/${id}/admins`, { headers: { @@ -497,7 +505,7 @@ class AddOnApiHelper { preferredEvents?: string[]; }, ): Promise { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const configuredWebhook = webhookUrl || webhookSecret || preferredEvents; @@ -531,7 +539,7 @@ class AddOnApiHelper { offset?: number; }, ) { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.get( `${(await getApiConfig()).SITE_ENDPOINT}/${siteId}/webhookLogs`, @@ -550,7 +558,7 @@ class AddOnApiHelper { } static async fetchAvailableWebhookEvents(siteId: string) { - const idToken = await this.getIdToken(); + const { idToken } = await this.getIdToken(); const resp = await axios.get( `${(await getApiConfig()).SITE_ENDPOINT}/${siteId}/availableWebhookEvents`,