Introduce pagesManifest (#3016)

docs/schemas/v1/definitions.json

@@ -422,7 +422,7 @@
               "description": "Authorization configuration for this page."
             },
             "unstable_codeFile": {
-              "type": "string",
+              "type": "boolean",
               "description": "The content of the page as JSX. Experimental, do not use!."
             },
             "display": {

packages/toolpad-app/src/appDom/index.ts

@@ -98,7 +98,7 @@ export interface PageNode extends AppDomNodeBase {
     readonly parameters?: [string, string][];
     readonly module?: string;
     readonly display?: PageDisplayMode;
-    readonly codeFile?: string;
+    readonly codeFile?: boolean;
     readonly displayName?: string;
     readonly authorization?: {
       readonly allowAll?: boolean;

packages/toolpad-app/src/server/appBuilderWorker.ts

@@ -24,6 +24,7 @@ async function main() {
     root: project.getRoot(),
     base,
     getComponents: () => project.getComponents(),
+    getPagesManifest: () => project.getPagesManifest(),
     outDir: project.getAppOutputFolder(),
     loadDom: () => project.loadDom(),
   });

packages/toolpad-app/src/server/appServerWorker.ts

@@ -5,7 +5,7 @@ import { createRpcClient } from '@mui/toolpad-utils/workerRpc';
 import { getHtmlContent, createViteConfig } from './toolpadAppBuilder';
 import type { RuntimeConfig } from '../types';
 import type * as appDom from '../appDom';
-import type { ComponentEntry } from './localMode';
+import type { ComponentEntry, PagesManifest } from './localMode';
 import createRuntimeState from '../runtime/createRuntimeState';
 import { postProcessHtml } from './toolpadAppServer';
 
@@ -15,9 +15,10 @@ export type WorkerRpc = {
   notifyReady: () => Promise<void>;
   loadDom: () => Promise<appDom.AppDom>;
   getComponents: () => Promise<ComponentEntry[]>;
+  getPagesManifest: () => Promise<PagesManifest>;
 };
 
-const { notifyReady, loadDom, getComponents } = createRpcClient<WorkerRpc>(
+const { notifyReady, loadDom, getComponents, getPagesManifest } = createRpcClient<WorkerRpc>(
   workerData.mainThreadRpcPort,
 );
 
@@ -75,6 +76,7 @@ export async function main({ port, ...config }: AppViteServerConfig) {
     dev: true,
     plugins: [devServerPlugin(config)],
     getComponents,
+    getPagesManifest,
     loadDom,
   });
 

packages/toolpad-app/src/server/index.ts

@@ -92,6 +92,7 @@ async function createDevHandler(project: ToolpadProject) {
     notifyReady: async () => resolveReadyPromise?.(),
     loadDom: async () => project.loadDom(),
     getComponents: async () => project.getComponents(),
+    getPagesManifest: async () => project.getPagesManifest(),
   });
 
   project.events.on('componentsListChanged', () => {

packages/toolpad-app/src/server/localMode.ts

@@ -11,6 +11,7 @@ import { glob } from 'glob';
 import * as chokidar from 'chokidar';
 import { debounce, throttle } from 'lodash-es';
 import { Emitter } from '@mui/toolpad-utils/events';
+import { guessTitle } from '@mui/toolpad-utils/strings';
 import { errorFrom } from '@mui/toolpad-utils/errors';
 import { filterValues, hasOwnProperty, mapValues } from '@mui/toolpad-utils/collections';
 import { execa } from 'execa';
@@ -124,7 +125,7 @@ export interface ComponentEntry {
 
 async function getComponents(root: string): Promise<ComponentEntry[]> {
   const componentsFolder = getComponentsFolder(root);
-  const entries = (await readMaybeDir(componentsFolder)) || [];
+  const entries = await readMaybeDir(componentsFolder);
   const result = entries.map((entry) => {
     if (entry.isFile()) {
       const fileName = entry.name;
@@ -151,7 +152,7 @@ async function loadCodeComponentsFromFiles(root: string): Promise<ComponentsCont
 
 async function loadPagesFromFiles(root: string): Promise<PagesContent> {
   const pagesFolder = getPagesFolder(root);
-  const entries = (await readMaybeDir(pagesFolder)) || [];
+  const entries = await readMaybeDir(pagesFolder);
   const resultEntries = await Promise.all(
     entries.map(async (entry): Promise<[string, Page] | null> => {
       if (entry.isDirectory()) {
@@ -197,16 +198,14 @@ async function loadPagesFromFiles(root: string): Promise<PagesContent> {
 
         for (const extension of extensions) {
           if (pageDirEntries.has(`page${extension}`)) {
-            const codeFileName = `./page${extension}`;
-
             return [
               pageName,
               {
                 apiVersion: API_VERSION,
                 kind: 'page',
                 spec: {
                   id: pageName,
-                  unstable_codeFile: codeFileName,
+                  unstable_codeFile: true,
                 },
               } satisfies Page,
             ];
@@ -1057,6 +1056,8 @@ class ToolpadProject {
 
   private pendingVersionCheck: Promise<VersionInfo> | undefined;
 
+  private pagesManifestPromise: Promise<PagesManifest> | undefined;
+
   constructor(root: string, options: ToolpadProjectOptions) {
     invariant(
       // eslint-disable-next-line no-underscore-dangle
@@ -1125,6 +1126,17 @@ class ToolpadProject {
     chokidar.watch(getDomFilePatterns(this.root), watchOptions).on('all', () => {
       updateDomFromExternal();
     });
+
+    chokidar
+      .watch([path.resolve(this.root, './pages/*/page.*')], watchOptions)
+      .on('all', async () => {
+        const oldManifest = await this.pagesManifestPromise;
+        this.pagesManifestPromise = buildPagesManifest(this.root);
+        const newManifest = await this.pagesManifestPromise;
+        if (JSON.stringify(oldManifest) !== JSON.stringify(newManifest)) {
+          this.events.emit('pagesManifestChanged', {});
+        }
+      });
   }
 
   private async loadDomAndFingerprint() {
@@ -1343,6 +1355,13 @@ class ToolpadProject {
       return null;
     }
   }
+
+  async getPagesManifest(): Promise<PagesManifest> {
+    if (!this.pagesManifestPromise) {
+      this.pagesManifestPromise = buildPagesManifest(this.root);
+    }
+    return this.pagesManifestPromise;
+  }
 }
 
 export type { ToolpadProject };
@@ -1372,3 +1391,78 @@ export async function initProject({ dir: dirInput, ...config }: InitProjectOptio
 
   return project;
 }
+
+const basePagesManifestEntrySchema = z.object({
+  slug: z.string(),
+  title: z.string(),
+  legacy: z.boolean().optional(),
+});
+
+export interface PagesManifestEntry extends z.infer<typeof basePagesManifestEntrySchema> {
+  children: PagesManifestEntry[];
+}
+
+const pagesManifestEntrySchema: z.ZodType<PagesManifestEntry> = basePagesManifestEntrySchema.extend(
+  {
+    children: z.array(z.lazy(() => pagesManifestEntrySchema)),
+  },
+);
+
+const pagesManifestSchema = z.object({
+  pages: z.array(pagesManifestEntrySchema),
+});
+
+export type PagesManifest = z.infer<typeof pagesManifestSchema>;
+
+async function buildPagesManifest(root: string): Promise<PagesManifest> {
+  const pagesFolder = getPagesFolder(root);
+  const pageDirs = await readMaybeDir(pagesFolder);
+  const pages = (
+    await Promise.all(
+      pageDirs.map(async (page) => {
+        if (page.isDirectory()) {
+          const pagePath = path.resolve(pagesFolder, page.name);
+          const title = guessTitle(page.name);
+
+          const extensions = ['.tsx', '.jsx'];
+
+          for (const extension of extensions) {
+            const pageFilePath = path.resolve(pagePath, `page${extension}`);
+
+            // eslint-disable-next-line no-await-in-loop
+            const stat = await fs.stat(pageFilePath).catch(() => null);
+            if (stat?.isFile()) {
+              return [
+                {
+                  slug: page.name,
+                  title,
+                  children: [],
+                },
+              ];
+            }
+          }
+
+          const pageFilePath = path.resolve(pagePath, 'page.yml');
+
+          const stat = await fs.stat(pageFilePath).catch(() => null);
+          if (stat?.isFile()) {
+            return [
+              {
+                slug: page.name,
+                title,
+                legacy: true,
+                children: [],
+              },
+            ];
+          }
+        }
+
+        return [];
+      }),
+    )
+  ).flat();
+
+  pages.sort((page1, page2) => page1.title.localeCompare(page2.title));
+
+  return { pages };
+}

packages/toolpad-app/src/server/schema.ts

@@ -313,8 +313,8 @@ export const pageSchema = toolpadObjectSchema(
       })
       .optional()
       .describe('Authorization configuration for this page.'),
-    unstable_codeFile: z
-      .string()
+    unstable_codeFile: z.coerce
+      .boolean()
       .optional()
       .describe('The content of the page as JSX. Experimental, do not use!.'),
     display: z

packages/toolpad-app/src/server/toolpadAppBuilder.ts

@@ -3,7 +3,7 @@ import * as url from 'node:url';
 import type { InlineConfig, Plugin } from 'vite';
 import react from '@vitejs/plugin-react';
 import { indent } from '@mui/toolpad-utils/strings';
-import type { ComponentEntry } from './localMode';
+import type { ComponentEntry, PagesManifest } from './localMode';
 import { INITIAL_STATE_WINDOW_PROPERTY } from '../constants';
 import * as appDom from '../appDom';
 import { pathToNodeImportSpecifier } from '../utils/paths';
@@ -143,6 +143,7 @@ export interface CreateViteConfigParams {
   plugins?: Plugin[];
   getComponents: () => Promise<ComponentEntry[]>;
   loadDom: () => Promise<appDom.AppDom>;
+  getPagesManifest: () => Promise<PagesManifest>;
 }
 
 export async function createViteConfig({
@@ -154,6 +155,7 @@ export async function createViteConfig({
   plugins = [],
   getComponents,
   loadDom,
+  getPagesManifest,
 }: CreateViteConfigParams) {
   const mode = dev ? 'development' : 'production';
 
@@ -229,7 +231,7 @@ if (import.meta.hot) {
     for (const page of pages) {
       const codeFile = page.attributes.codeFile;
       if (codeFile) {
-        const importPath = path.resolve(root, `./pages/${page.name}`, codeFile);
+        const importPath = path.resolve(root, `./pages/${page.name}/page`);
         const relativeImportPath = path.relative(root, importPath);
         const importSpec = `toolpad-user-project:${pathToNodeImportSpecifier(relativeImportPath)}`;
         imports.set(page.name, importSpec);
@@ -260,6 +262,7 @@ if (import.meta.hot) {
     ['canvas.tsx', getEntryPoint(true)],
     ['components.tsx', await createComponentsFile()],
     ['page-components.tsx', await createPageComponentsFile()],
+    ['pages-manifest.json', JSON.stringify(await getPagesManifest(), null, 2)],
   ]);
 
   const virtualToolpadFiles = viteVirtualPlugin(virtualFiles, 'toolpad-files');
@@ -391,6 +394,7 @@ export interface ToolpadBuilderParams {
   outDir: string;
   getComponents: () => Promise<ComponentEntry[]>;
   loadDom: () => Promise<appDom.AppDom>;
+  getPagesManifest: () => Promise<PagesManifest>;
   root: string;
   base: string;
 }
@@ -399,6 +403,7 @@ export async function buildApp({
   root,
   base,
   getComponents,
+  getPagesManifest,
   loadDom,
   outDir,
 }: ToolpadBuilderParams) {
@@ -408,6 +413,7 @@ export async function buildApp({
     base,
     outDir,
     getComponents,
+    getPagesManifest,
     loadDom,
   });
   const vite = await import('vite');

packages/toolpad-app/src/types.ts

@@ -200,6 +200,8 @@ export type ProjectEvents = {
   envChanged: {};
   // Functions or datasources have been updated
   functionsChanged: {};
+  // Pagesmanifest has changed
+  pagesManifestChanged: {};
 };
 
 export interface ToolpadProjectOptions {

packages/toolpad-utils/src/fs.ts

@@ -32,20 +32,20 @@ export async function readMaybeFile(filePath: string): Promise<string | null> {
     return await fs.readFile(filePath, { encoding: 'utf-8' });
   } catch (rawError) {
     const error = errorFrom(rawError);
-    if (error.code === 'ENOENT') {
+    if (error.code === 'ENOENT' || error.code === 'EISDIR') {
       return null;
     }
     throw error;
   }
 }
 
-export async function readMaybeDir(dirPath: string): Promise<Dirent[] | null> {
+export async function readMaybeDir(dirPath: string): Promise<Dirent[]> {
   try {
     return await fs.readdir(dirPath, { withFileTypes: true });
   } catch (rawError: unknown) {
     const error = errorFrom(rawError);
-    if (errorFrom(error).code === 'ENOENT') {
-      return null;
+    if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
+      return [];
     }
     throw error;
   }