Update script to support current tailwindui.com

.env.sample

@@ -0,0 +1,5 @@
+BASE_URL=https://tailwindui.com
+
+email=some-email
+password=some-password
+slowmo=0

README.md

@@ -26,10 +26,10 @@ The two steps process makes it easy to adjust for future needs.
 
    ```sh
    # Format:
-   email=<[email protected]> password=<yourpassword> node tailwindui.js <react|vue|html>
+   yarn start <react|vue|html>
 
    # Example:
-   email='[email protected]' password='pass123' node tailwindui.js react
+   yarn start react
    ```
 
    The command will start chromium and navigate thru tailwindui.com website to copy the components codes. Here's how it looks like:
@@ -40,8 +40,8 @@ The two steps process makes it easy to adjust for future needs.
 5. Now we can create the mdx files based on above json. To do so, run:
 
    ```sh
-   # Replace `react.json` suffix with `vue.json` as you fit
-   node tailwindui-storybook.js ./output/tailwindui.react.json react
+    # Replace "react" with "html" or "vue" (based on step 3)
+   yarn create-mdx react
    ```
 
    ![tailwindui-storybook](docs/tailwindui-storybook-mdx.gif)

lib/getComponents.js

@@ -0,0 +1,46 @@
+const baseUrl = process.env.BASE_URL;
+
+async function getComponents(page, sectionUrl, type = "html") {
+  // Go to section url. Example: https://tailwindui.com/components/marketing/sections/heroes
+  await page.goto(sectionUrl);
+
+  const toggleCodeSelector = '[role=tablist] > [id^=headlessui-tabs-tab-]:nth-child(2)'
+  try {
+    // Element to toggle code snippet (hint: ../docs/hints/toggle-code.png). 
+    // The element won't exist if the user doesn't have access to the paid component.
+    await page.waitForSelector(toggleCodeSelector, { timeout: 10 * 1000 });
+  } catch (error) {
+    console.warn(`You do not have access to this section components: ${sectionUrl}`);
+  }
+
+  const components = await page.evaluate(([toggleCodeSelector, type]) => {
+    let components = [];
+    document.querySelectorAll('section[id^="component-"]').forEach((el) => {
+      const toggleCode = el.querySelector(toggleCodeSelector);
+      if (!toggleCode) {
+        return
+      }
+
+      // Toggle code block
+      el.querySelector(toggleCodeSelector)?.click();
+
+      // Change the type of code snippet to HTML/React/Vue (hint: ../docs/hints/toggle-code-format.png)
+      const toggleSnippetType = el.querySelector('select.form-select')
+      toggleSnippetType.value = type;
+      toggleSnippetType.dispatchEvent(new Event('change', { bubbles: true }));
+
+      // Extract code blocks (hint: ../docs/hints/codeblock.png)
+      const title = el.querySelector("h2 a").innerText;
+      const codeblock = el.querySelector('pre code')?.innerText;
+      codeblock && components.push({ title, codeblocks: { [type]: codeblock } });
+    });
+    return Promise.resolve(components);
+  }, [toggleCodeSelector, type]);
+
+  // Back to home page
+  await page.goto(baseUrl);
+
+  return components;
+}
+
+module.exports = getComponents

lib/getSections.js

@@ -0,0 +1,18 @@
+async function getSections(page) {
+  const sections = await page.evaluate(([]) => {
+    const buildSections = [];
+    document
+      .querySelectorAll('a[href^="/components"]')
+      .forEach((el) => {
+        const title = el.children[1]?.innerText;
+        const countComponents = el.parentElement?.nextElementSibling?.innerText;
+        const url = el.href;
+        title && countComponents && buildSections.push({ title, componentsCount: parseInt(countComponents), url });
+      });
+    return Promise.resolve(buildSections);
+  }, []);
+
+  return sections;
+}
+
+module.exports = getSections

lib/getSections.sample.json

@@ -0,0 +1,12 @@
+[
+  {
+    "title": "Hero Sections",
+    "componentsCount": 9,
+    "url": "https://tailwindui.com/components/marketing/sections/heroes"
+  },
+  {
+    "title": "Feature Sections",
+    "componentsCount": 10,
+    "url": "https://tailwindui.com/components/marketing/sections/feature-sections"
+  }
+]

lib/login.js

@@ -0,0 +1,22 @@
+const baseUrl = process.env.BASE_URL;
+const selector = {
+  email: 'input#email',
+  password: 'input#password',
+  submit: 'button[type=submit]',
+}
+
+async function login(page, email, password) {
+  await page.goto(baseUrl + "/login");
+  await page.locator(selector.email).fill(email);
+  await page.locator(selector.password).fill(password);
+  await page.locator(selector.submit).click();
+
+  // Assert login succeeded
+  const loginFailedToken = "These credentials do not match our records";
+  const el = await page.$$(`:text("${loginFailedToken}")`);
+  if (el.length) {
+    throw new Error("invalid credentials");
+  }
+}
+
+module.exports = login

package.json

@@ -1,7 +1,13 @@
 {
   "name": "tailwindui-storybook",
+  "engine": ">=14",
+  "scripts": {
+    "start": "node -r dotenv/config tailwindui.js",
+    "create-mdx": "node -r dotenv/config tailwindui-storybook.js"
+  },
   "dependencies": {
+    "dotenv": "^16.0.3",
     "lodash": "^4.17.21",
-    "playwright": "^1.9.2"
+    "playwright": "^1.27.1"
   }
 }

tailwindui-storybook.js

@@ -2,36 +2,31 @@ const fs = require("fs");
 const camelCase = require("lodash/camelCase");
 const startCase = require("lodash/startCase");
 
-const [, , jsonFile, componentType] = process.argv;
+const createDir = require("./util/createDir");
 
-if (!jsonFile || !componentType) {
-  console.log("usage: tailwind-storybook.js <jsonfile> <type>");
-  console.log("example: tailwind-storybook.js tailwindui.react.json react");
+const componentType = process.argv[process.argv.length - 1];
+if (!componentType.match(/react|html|vue/)) {
+  console.log("usage: tailwind-storybook.js <react|html|vue>");
+  console.log("example: tailwind-storybook.js react");
   process.exit(0);
 }
 
-const sections = require(jsonFile);
-
-const outputDir = `output/tailwindui-${componentType}`;
-
-const storyFolder = `Tailwind UI`;
-
-function createDir(dir) {
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir);
-    console.log("[INFO] directory created:", dir);
-  }
+const jsonFile = `./output/tailwindui.${componentType}.json`
+if (!fs.existsSync(jsonFile)) {
+  console.error("[ERROR] could not find json file at", jsonFile)
+  console.log(`→ Have you run 'yarn start ${componentType}'?`)
+  process.exit(1)
 }
 
-function pascalCase(str) {
-  return startCase(camelCase(str)).replace(/ /g, "");
-}
+const sections = require(jsonFile)
+const storyFolder = `Tailwind UI`;
+const outputDir = `output/tailwindui-${componentType}`;
 
 async function run() {
   createDir(outputDir);
 
   // Uncomment to see data structure
-  console.log("sections[0]:", sections[0]);
+  // console.log("sections[0]:", sections[0]);
 
   sections.forEach((section) => {
     const { title, components } = section;
@@ -84,4 +79,8 @@ ${componentStories.join("\n")}
   });
 }
 
+function pascalCase(str) {
+  return startCase(camelCase(str)).replace(/ /g, "");
+}
+
 run();

tailwindui.js

@@ -1,13 +1,23 @@
-const fs = require("fs");
 const playwright = require("playwright");
 
-const baseUrl = "https://tailwindui.com";
+const login = require("./lib/login");
+const getSections = require("./lib/getSections");
+const getComponents = require("./lib/getComponents");
+const createDir = require("./util/createDir");
+const writeToFile = require("./util/writeToFile");
 
 const email = (process.env.email || "").trim();
 const password = (process.env.password || "").trim();
 
+const outputDir = "output";
+const componentsPage = `${process.env.BASE_URL}/components`;
+
+const guestMode = !email && !password;
+
 // Make sure email and password are provided
-if (!email || !password) {
+if (guestMode) {
+  console.log("[INFO] using guest mode because both email and password are not provided")
+} else if (!email || !password) {
   console.log(
     "[ERROR] please provide email and password of your tailwind ui account as environment variable."
   );
@@ -17,115 +27,31 @@ if (!email || !password) {
   process.exit(1);
 }
 
-const outputDir = "output";
-
-async function login(page, email, password) {
-  await page.goto(baseUrl + "/login");
-  await page.fill('[name="email"]', email);
-  await page.fill('[name="password"]', password);
-  await page.click('[type="submit"]');
-
-  // Assert login succeeded
-  const loginFailedToken = "These credentials do not match our records";
-  const el = await page.$$(`:text("${loginFailedToken}")`);
-  if (el.length) {
-    throw new Error("invalid credentials");
-  }
-}
-
-async function getSections(page) {
-  const sections = await page.evaluate(([]) => {
-    let sections = [];
-    document
-      .querySelectorAll('#components [href^="/components"]')
-      .forEach((el) => {
-        const title = el.querySelector("p:nth-child(1)").innerText;
-        const components = el.querySelector("p:nth-child(2)").innerText;
-        const url = el.href;
-        sections.push({ title, componentsCount: parseInt(components), url });
-      });
-    return Promise.resolve(sections);
-  }, []);
-
-  return sections;
-}
-
-async function getComponents(page, sectionUrl, type = "html") {
-  // Go to section url
-  await page.goto(sectionUrl);
-
-  // Select react code
-  await page.waitForSelector('[x-model="activeSnippet"]');
-  await page.evaluate(
-    ([type]) => {
-      document.querySelectorAll('[x-model="activeSnippet"]').forEach((el) => {
-        el.value = type;
-        const evt = document.createEvent("HTMLEvents");
-        evt.initEvent("change", false, true);
-        el.dispatchEvent(evt);
-      });
-    },
-    [type]
-  );
-
-  // Toggle all codeblocks (otherwise the `innerText` will be empty)
-  await page.waitForSelector('[x-ref="code"]');
-  await page.evaluate(([]) => {
-    document.querySelectorAll('section[id^="component-"]').forEach((el) => {
-      el.querySelector('[x-ref="code"]').click();
-    });
-  }, []);
-
-  // Wait until codeblock is visible
-  await page.waitForSelector(`[x-ref="codeBlock${type}"]`);
-
-  const components = await page.evaluate(
-    ([type]) => {
-      let components = [];
-      document.querySelectorAll('section[id^="component-"]').forEach((el) => {
-        const title = el.querySelector("header h2").innerText;
-
-        const component = el.querySelector(`[x-ref="codeBlock${type}"]`)
-          .innerText;
-        components.push({ title, codeblocks: { [type]: component } });
-      });
-      return Promise.resolve(components);
-    },
-    [type]
-  );
-
-  // Back to home page
-  await page.goto(baseUrl);
-
-  return components;
-}
-
 async function run() {
   const [, , componentType] = process.argv;
+  createDir(outputDir);
 
   if (!componentType) {
-    console.error("[ERROR] please specify component type (html/react/vue).");
+    console.error("[ERROR] please specify component type (html|react|vue).");
     console.error(`example: node tailwindui.js react`);
     process.exit(0);
   }
 
-  if (!fs.existsSync(outputDir)) {
-    fs.mkdirSync(outputDir);
-    console.log("[INFO] output directory created");
-  }
-
   const browser = await playwright["chromium"].launch({
-    headless: false,
-    // slowMo: 200, // Uncomment to activate slow mo
+    headless: !!process.env.headless,
+    slowMo: process.env.slowmo ? 500 : undefined,
   });
   const ctx = await browser.newContext();
   const page = await ctx.newPage();
 
-  console.log("[INFO] logging in to tailwindui.com..");
 
   try {
-    // Login to tailwindui.com. Throws error if failed.
-    await login(page, email, password);
+    // Login to tailwindui.com if not in guest mode. Throws error if failed.
+    if (!guestMode) {
+      console.log("[INFO] logging in to tailwindui.com..");
+      await login(page, email, password);
+    }
+    await page.goto(componentsPage);
   } catch (e) {
     const maskPassword = password.replace(/.{4}$/g, "*".repeat(4));
     console.log(
@@ -135,7 +61,7 @@ async function run() {
   }
 
   console.log(`[INFO] fetching sections..`);
-  let sections = await getSections(page).catch((e) => {
+  const sections = await getSections(page).catch((e) => {
     console.log("[ERROR] getSections failed:", e);
   });
 
@@ -150,8 +76,7 @@ async function run() {
   for (const i in sections) {
     const { title, componentsCount, url } = sections[i];
     console.log(
-      `[${parseInt(i) + 1}/${
-        sections.length
+      `[${parseInt(i) + 1}/${sections.length
       }] fetching ${componentType} components: ${title} (${componentsCount} components)`
     );
 
@@ -160,6 +85,7 @@ async function run() {
       components = await getComponents(page, url, componentType);
     } catch (e) {
       console.log("[ERROR] getComponent failed:", title, e.message);
+      writeToFile(outputDir, componentType, sections, true);
       process.exit(1);
     }
 
@@ -168,11 +94,7 @@ async function run() {
 
   await browser.close();
 
-  const jsonFile = `${outputDir}/tailwindui.${componentType}.json`;
-  console.log("[INFO] writing json file:", jsonFile);
-
-  fs.writeFileSync(jsonFile, JSON.stringify(sections));
-
+  writeToFile(outputDir, componentType, sections);
   console.log("[INFO] done!");
 }