From 350981baa5b19367631b60f9b819ffcfe37f9d8f Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 12:34:07 +0100 Subject: [PATCH 1/8] Remove functions worker --- .../src/server/FunctionsManager.ts | 37 +-- .../src/server/functionsDevWorker.ts | 265 ------------------ .../src/server/functionsRuntime.ts | 131 +++++++++ packages/toolpad-app/tsup.config.ts | 1 - packages/toolpad-core/package.json | 1 + packages/toolpad-core/src/serverRuntime.ts | 10 + 6 files changed, 148 insertions(+), 297 deletions(-) delete mode 100644 packages/toolpad-app/src/server/functionsDevWorker.ts create mode 100644 packages/toolpad-app/src/server/functionsRuntime.ts diff --git a/packages/toolpad-app/src/server/FunctionsManager.ts b/packages/toolpad-app/src/server/FunctionsManager.ts index a4111e8f8d8..45b80a53bd0 100644 --- a/packages/toolpad-app/src/server/FunctionsManager.ts +++ b/packages/toolpad-app/src/server/FunctionsManager.ts @@ -17,11 +17,10 @@ import { import { errorFrom } from '@mui/toolpad-utils/errors'; import { ToolpadDataProviderIntrospection } from '@mui/toolpad-core/runtime'; import * as url from 'node:url'; -import invariant from 'invariant'; import type { GridRowId } from '@mui/x-data-grid'; import EnvManager from './EnvManager'; import { ProjectEvents, ToolpadProjectOptions } from '../types'; -import { createWorker as createDevWorker } from './functionsDevWorker'; +import * as functionsRuntime from './functionsRuntime'; import type { ExtractTypesParams, IntrospectionResult } from './functionsTypesWorker'; import { Awaitable } from '../utils/types'; import { format } from '../utils/prettier'; @@ -114,8 +113,6 @@ export default class FunctionsManager { private buildErrors: esbuild.Message[] = []; - private devWorker: ReturnType | undefined; - private extractedTypes: Awaitable | undefined; private extractTypesWorker: Piscina | undefined; @@ -252,24 +249,10 @@ export default class FunctionsManager { resourcesWatcher.on('unlink', reinitializeWatcher); } - private async createRuntimeWorker() { - const oldWorker = this.devWorker; - this.devWorker = createDevWorker(this.project.envManager.getEnv()); - await oldWorker?.terminate(); - this.project.invalidateQueries(); - } - async start() { - await this.createRuntimeWorker(); - if (this.project.options.dev) { await this.migrateLegacy(); - await this.startWatchingFunctionFiles(); - - this.project.events.subscribe('envChanged', async () => { - await this.createRuntimeWorker(); - }); } } @@ -293,11 +276,7 @@ export default class FunctionsManager { } async dispose() { - await Promise.all([ - this.disposeBuildcontext(), - this.devWorker?.terminate(), - this.extractTypesWorker?.destroy(), - ]); + await Promise.all([this.disposeBuildcontext(), this.extractTypesWorker?.destroy()]); } async getBuiltOutputFilePath(fileName: string): Promise { @@ -351,8 +330,7 @@ export default class FunctionsManager { ): Promise> { const outputFilePath = await this.getBuiltOutputFilePath(fileName); - invariant(this.devWorker, 'devWorker must be initialized'); - const data = await this.devWorker.execute(outputFilePath, name, parameters); + const data = await functionsRuntime.execute(outputFilePath, name, parameters); return { data }; } @@ -396,8 +374,7 @@ export default class FunctionsManager { exportName: string = 'default', ): Promise { const fullPath = await this.getBuiltOutputFilePath(fileName); - invariant(this.devWorker, 'devWorker must be initialized'); - return this.devWorker.introspectDataProvider(fullPath, exportName); + return functionsRuntime.introspectDataProvider(fullPath, exportName); } async getDataProviderRecords( @@ -406,8 +383,7 @@ export default class FunctionsManager { params: GetRecordsParams, ): Promise> { const fullPath = await this.getBuiltOutputFilePath(fileName); - invariant(this.devWorker, 'devWorker must be initialized'); - return this.devWorker.getDataProviderRecords(fullPath, exportName, params); + return functionsRuntime.getDataProviderRecords(fullPath, exportName, params); } async deleteDataProviderRecord( @@ -416,7 +392,6 @@ export default class FunctionsManager { id: GridRowId, ): Promise { const fullPath = await this.getBuiltOutputFilePath(fileName); - invariant(this.devWorker, 'devWorker must be initialized'); - return this.devWorker.deleteDataProviderRecord(fullPath, exportName, id); + return functionsRuntime.deleteDataProviderRecord(fullPath, exportName, id); } } diff --git a/packages/toolpad-app/src/server/functionsDevWorker.ts b/packages/toolpad-app/src/server/functionsDevWorker.ts deleted file mode 100644 index 2d5cfa6b7ee..00000000000 --- a/packages/toolpad-app/src/server/functionsDevWorker.ts +++ /dev/null @@ -1,265 +0,0 @@ -import { Worker, MessageChannel, isMainThread, parentPort } from 'worker_threads'; -import * as path from 'path'; -import { createRequire } from 'node:module'; -import * as fs from 'fs/promises'; -import * as vm from 'vm'; -import * as url from 'node:url'; -import { - ServerContext, - getServerContext, - initialContextStore, - withContext, -} from '@mui/toolpad-core/serverRuntime'; -import { isWebContainer } from '@webcontainer/env'; -import * as superjson from 'superjson'; -import { createRpcClient, serveRpc } from '@mui/toolpad-utils/workerRpc'; -import { workerData } from 'node:worker_threads'; -import { ToolpadDataProviderIntrospection } from '@mui/toolpad-core/runtime'; -import { TOOLPAD_DATA_PROVIDER_MARKER, ToolpadDataProvider } from '@mui/toolpad-core/server'; -import * as z from 'zod'; -import { fromZodError } from 'zod-validation-error'; -import { GetRecordsParams, GetRecordsResult, PaginationMode } from '@mui/toolpad-core'; -import invariant from 'invariant'; -import type { GridRowId } from '@mui/x-data-grid'; - -import.meta.url ??= url.pathToFileURL(__filename).toString(); -const currentDirectory = url.fileURLToPath(new URL('.', import.meta.url)); - -interface ModuleObject { - exports: Record; -} - -const fileContents = new Map(); -const moduleCache = new Map(); - -function loadModule(fullPath: string, content: string) { - const moduleRequire = createRequire(url.pathToFileURL(fullPath)); - const moduleObject: ModuleObject = { exports: {} }; - - const toolpadServer = moduleRequire('@mui/toolpad/server'); - // eslint-disable-next-line no-underscore-dangle - toolpadServer.__initContextStore(initialContextStore); - - vm.runInThisContext(`((require, exports, module) => {\n${content}\n})`)( - moduleRequire, - moduleObject.exports, - moduleObject, - ); - - return moduleObject; -} - -async function resolveExports(filePath: string): Promise> { - const fullPath = path.resolve(filePath); - const content = await fs.readFile(fullPath, 'utf-8'); - - if (content !== fileContents.get(fullPath)) { - moduleCache.delete(fullPath); - fileContents.set(fullPath, content); - } - - let cachedModule = moduleCache.get(fullPath); - - if (!cachedModule) { - cachedModule = loadModule(fullPath, content); - moduleCache.set(fullPath, cachedModule); - } - - return new Map(Object.entries(cachedModule.exports)); -} - -interface ExecuteParams { - filePath: string; - name: string; - parameters: unknown[]; - cookies?: Record; -} - -interface ExecuteResult { - result: string; - newCookies: [string, string][]; -} - -async function execute(msg: ExecuteParams): Promise { - const exports = await resolveExports(msg.filePath); - - const fn = exports.get(msg.name); - if (typeof fn !== 'function') { - throw new Error(`Function "${msg.name}" not found`); - } - - let functionFinished = false; - - try { - const newCookies = new Map(); - - const ctx: ServerContext = { - cookies: msg.cookies || {}, - setCookie(name: string, value: string) { - if (functionFinished) { - throw new Error(`setCookie can't be called after the function has finished executing.`); - } - newCookies.set(name, value); - }, - }; - - const shouldBypassContext = isWebContainer(); - - if (shouldBypassContext) { - console.warn( - 'Bypassing server context in web containers, see https://github.com/stackblitz/core/issues/2711', - ); - } - - const rawResult = shouldBypassContext - ? await fn(...msg.parameters) - : await withContext(ctx, async () => fn(...msg.parameters)); - - const serializedResult = superjson.stringify(rawResult); - - return { result: serializedResult, newCookies: Array.from(newCookies.entries()) }; - } finally { - functionFinished = true; - } -} - -const dataProviderSchema: z.ZodType> = z.object({ - paginationMode: z.enum(['index', 'cursor']).optional().default('index'), - getRecords: z.function(z.tuple([z.any()]), z.any()), - deleteRecord: z.function(z.tuple([z.any()]), z.any()).optional(), - updateRecord: z.function(z.tuple([z.any()]), z.any()).optional(), - createRecord: z.function(z.tuple([z.any()]), z.any()).optional(), - [TOOLPAD_DATA_PROVIDER_MARKER]: z.literal(true), -}); - -async function loadDataProvider( - filePath: string, - name: string, -): Promise> { - const exports = await resolveExports(filePath); - const dataProviderExport = exports.get(name); - - if (!dataProviderExport || typeof dataProviderExport !== 'object') { - throw new Error(`DataProvider "${name}" not found`); - } - - const parsed = dataProviderSchema.safeParse(dataProviderExport); - - if (parsed.success) { - return parsed.data; - } - - throw fromZodError(parsed.error); -} - -async function introspectDataProvider( - filePath: string, - name: string, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - - return { - paginationMode: dataProvider.paginationMode, - hasDeleteRecord: !!dataProvider.deleteRecord, - }; -} - -async function getDataProviderRecords( - filePath: string, - name: string, - params: GetRecordsParams, -): Promise> { - const dataProvider = await loadDataProvider(filePath, name); - - return dataProvider.getRecords(params); -} - -async function deleteDataProviderRecord( - filePath: string, - name: string, - id: GridRowId, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - invariant(dataProvider.deleteRecord, 'DataProvider does not support deleteRecord'); - return dataProvider.deleteRecord(id); -} - -type WorkerRpcServer = { - execute: typeof execute; - introspectDataProvider: typeof introspectDataProvider; - getDataProviderRecords: typeof getDataProviderRecords; - deleteDataProviderRecord: typeof deleteDataProviderRecord; -}; - -if (!isMainThread && parentPort) { - serveRpc(workerData.workerRpcPort, { - execute, - introspectDataProvider, - getDataProviderRecords, - deleteDataProviderRecord, - }); -} - -export function createWorker(env: Record) { - const workerRpcChannel = new MessageChannel(); - const worker = new Worker(path.resolve(currentDirectory, '../cli/functionsDevWorker.mjs'), { - env, - workerData: { - workerRpcPort: workerRpcChannel.port1, - }, - transferList: [workerRpcChannel.port1], - }); - - const client = createRpcClient(workerRpcChannel.port2); - - return { - async terminate() { - return worker.terminate(); - }, - - async execute(filePath: string, name: string, parameters: unknown[]): Promise { - const ctx = getServerContext(); - - const { result: serializedResult, newCookies } = await client.execute({ - filePath, - name, - parameters, - cookies: ctx?.cookies, - }); - - if (ctx) { - for (const [cookieName, cookieValue] of newCookies) { - ctx.setCookie(cookieName, cookieValue); - } - } - - const result = superjson.parse(serializedResult); - - return result; - }, - - async introspectDataProvider( - filePath: string, - name: string, - ): Promise { - return client.introspectDataProvider(filePath, name); - }, - - async getDataProviderRecords( - filePath: string, - name: string, - params: GetRecordsParams, - ): Promise> { - return client.getDataProviderRecords(filePath, name, params); - }, - - async deleteDataProviderRecord(filePath: string, name: string, id: GridRowId): Promise { - return client.deleteDataProviderRecord(filePath, name, id); - }, - }; -} - -process.on('unhandledRejection', (error) => { - console.error(error); - process.exit(1); -}); diff --git a/packages/toolpad-app/src/server/functionsRuntime.ts b/packages/toolpad-app/src/server/functionsRuntime.ts new file mode 100644 index 00000000000..0fcf7f6b81d --- /dev/null +++ b/packages/toolpad-app/src/server/functionsRuntime.ts @@ -0,0 +1,131 @@ +import * as path from 'path'; +import { createRequire } from 'node:module'; +import * as fs from 'fs/promises'; +import * as vm from 'vm'; +import * as url from 'node:url'; +import { initialContextStore } from '@mui/toolpad-core/serverRuntime'; +import { ToolpadDataProviderIntrospection } from '@mui/toolpad-core/runtime'; +import { TOOLPAD_DATA_PROVIDER_MARKER, ToolpadDataProvider } from '@mui/toolpad-core/server'; +import * as z from 'zod'; +import { fromZodError } from 'zod-validation-error'; +import { GetRecordsParams, GetRecordsResult, PaginationMode } from '@mui/toolpad-core'; +import invariant from 'invariant'; +import type { GridRowId } from '@mui/x-data-grid'; + +import.meta.url ??= url.pathToFileURL(__filename).toString(); + +interface ModuleObject { + exports: Record; +} + +const fileContents = new Map(); +const moduleCache = new Map(); + +function loadModule(fullPath: string, content: string) { + const moduleRequire = createRequire(url.pathToFileURL(fullPath)); + const moduleObject: ModuleObject = { exports: {} }; + + const serverRuntime = moduleRequire('@mui/toolpad-core/serverRuntime'); + serverRuntime.initStore(initialContextStore); + + vm.runInThisContext(`((require, exports, module) => {\n${content}\n})`)( + moduleRequire, + moduleObject.exports, + moduleObject, + ); + + return moduleObject; +} + +async function resolveExports(filePath: string): Promise> { + const fullPath = path.resolve(filePath); + const content = await fs.readFile(fullPath, 'utf-8'); + + if (content !== fileContents.get(fullPath)) { + moduleCache.delete(fullPath); + fileContents.set(fullPath, content); + } + + let cachedModule = moduleCache.get(fullPath); + + if (!cachedModule) { + cachedModule = loadModule(fullPath, content); + moduleCache.set(fullPath, cachedModule); + } + + return new Map(Object.entries(cachedModule.exports)); +} + +const dataProviderSchema: z.ZodType> = z.object({ + paginationMode: z.enum(['index', 'cursor']).optional().default('index'), + getRecords: z.function(z.tuple([z.any()]), z.any()), + deleteRecord: z.function(z.tuple([z.any()]), z.any()).optional(), + updateRecord: z.function(z.tuple([z.any()]), z.any()).optional(), + createRecord: z.function(z.tuple([z.any()]), z.any()).optional(), + [TOOLPAD_DATA_PROVIDER_MARKER]: z.literal(true), +}); + +async function loadDataProvider( + filePath: string, + name: string, +): Promise> { + const exports = await resolveExports(filePath); + const dataProviderExport = exports.get(name); + + if (!dataProviderExport || typeof dataProviderExport !== 'object') { + throw new Error(`DataProvider "${name}" not found`); + } + + const parsed = dataProviderSchema.safeParse(dataProviderExport); + + if (parsed.success) { + return parsed.data; + } + + throw fromZodError(parsed.error); +} + +export async function introspectDataProvider( + filePath: string, + name: string, +): Promise { + const dataProvider = await loadDataProvider(filePath, name); + + return { + paginationMode: dataProvider.paginationMode, + hasDeleteRecord: !!dataProvider.deleteRecord, + }; +} + +export async function getDataProviderRecords( + filePath: string, + name: string, + params: GetRecordsParams, +): Promise> { + const dataProvider = await loadDataProvider(filePath, name); + + return dataProvider.getRecords(params); +} + +export async function deleteDataProviderRecord( + filePath: string, + name: string, + id: GridRowId, +): Promise { + const dataProvider = await loadDataProvider(filePath, name); + invariant(dataProvider.deleteRecord, 'DataProvider does not support deleteRecord'); + return dataProvider.deleteRecord(id); +} + +export async function execute(filePath: string, name: string, parameters: unknown[]): Promise { + const exports = await resolveExports(filePath); + + const fn = exports.get(name); + if (typeof fn !== 'function') { + throw new Error(`Function "${name}" not found`); + } + + const result = await fn(...parameters); + + return result; +} diff --git a/packages/toolpad-app/tsup.config.ts b/packages/toolpad-app/tsup.config.ts index a5f6b5f9e37..3f7b1166247 100644 --- a/packages/toolpad-app/tsup.config.ts +++ b/packages/toolpad-app/tsup.config.ts @@ -25,7 +25,6 @@ export default defineConfig((options) => [ // Worker entry points appServerWorker: './src/server/appServerWorker.ts', appBuilderWorker: './src/server/appBuilderWorker.ts', - functionsDevWorker: './src/server/functionsDevWorker.ts', functionsTypesWorker: './src/server/functionsTypesWorker.ts', }, format: ['esm'], diff --git a/packages/toolpad-core/package.json b/packages/toolpad-core/package.json index 94f51525a91..740b1497abe 100644 --- a/packages/toolpad-core/package.json +++ b/packages/toolpad-core/package.json @@ -44,6 +44,7 @@ "@mui/toolpad-utils": "0.1.41", "@tanstack/react-query": "5.14.1", "@types/json-schema": "7.0.15", + "@webcontainer/env": "1.1.0", "cookie": "0.6.0", "invariant": "2.2.4", "quickjs-emscripten": "0.24.0", diff --git a/packages/toolpad-core/src/serverRuntime.ts b/packages/toolpad-core/src/serverRuntime.ts index a6a097518b3..9b1cdaef94a 100644 --- a/packages/toolpad-core/src/serverRuntime.ts +++ b/packages/toolpad-core/src/serverRuntime.ts @@ -1,6 +1,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import { IncomingMessage, ServerResponse } from 'node:http'; import * as cookie from 'cookie'; +import { isWebContainer } from '@webcontainer/env'; export interface ServerContext { /** @@ -40,5 +41,14 @@ export function createServerContext(req: IncomingMessage, res: ServerResponse): } export function withContext(ctx: ServerContext, doWork: () => Promise): Promise { + const shouldBypassContext = isWebContainer(); + + if (shouldBypassContext) { + console.warn( + 'Bypassing server context in web containers, see https://github.com/stackblitz/core/issues/2711', + ); + return doWork(); + } + return contextStore.run(ctx, doWork); } From 4ca88463ae58c9491e2a5e9c7371b4fb61aa09b9 Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 12:46:51 +0100 Subject: [PATCH 2/8] Update functionsRuntime.ts --- packages/toolpad-app/src/server/functionsRuntime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/toolpad-app/src/server/functionsRuntime.ts b/packages/toolpad-app/src/server/functionsRuntime.ts index 0fcf7f6b81d..a1b7d6ae57b 100644 --- a/packages/toolpad-app/src/server/functionsRuntime.ts +++ b/packages/toolpad-app/src/server/functionsRuntime.ts @@ -26,7 +26,7 @@ function loadModule(fullPath: string, content: string) { const moduleObject: ModuleObject = { exports: {} }; const serverRuntime = moduleRequire('@mui/toolpad-core/serverRuntime'); - serverRuntime.initStore(initialContextStore); + serverRuntime.__initContextStore(initialContextStore); vm.runInThisContext(`((require, exports, module) => {\n${content}\n})`)( moduleRequire, From e23b1b89a128e2dfc6423d6252cc1a81f166e783 Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 12:48:09 +0100 Subject: [PATCH 3/8] Update functionsRuntime.ts --- packages/toolpad-app/src/server/functionsRuntime.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/toolpad-app/src/server/functionsRuntime.ts b/packages/toolpad-app/src/server/functionsRuntime.ts index a1b7d6ae57b..e6b26013dc8 100644 --- a/packages/toolpad-app/src/server/functionsRuntime.ts +++ b/packages/toolpad-app/src/server/functionsRuntime.ts @@ -25,7 +25,8 @@ function loadModule(fullPath: string, content: string) { const moduleRequire = createRequire(url.pathToFileURL(fullPath)); const moduleObject: ModuleObject = { exports: {} }; - const serverRuntime = moduleRequire('@mui/toolpad-core/serverRuntime'); + const serverRuntime = moduleRequire('@mui/toolpad/server'); + // eslint-disable-next-line no-underscore-dangle serverRuntime.__initContextStore(initialContextStore); vm.runInThisContext(`((require, exports, module) => {\n${content}\n})`)( From 921cc320b0a7d4f0beff3c0318d604b91c89364e Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:16:33 +0100 Subject: [PATCH 4/8] trim more fat --- .../src/server/FunctionsManager.ts | 14 ++++-- .../src/server/functionsRuntime.ts | 44 ++----------------- 2 files changed, 15 insertions(+), 43 deletions(-) diff --git a/packages/toolpad-app/src/server/FunctionsManager.ts b/packages/toolpad-app/src/server/FunctionsManager.ts index 45b80a53bd0..eb3ace12628 100644 --- a/packages/toolpad-app/src/server/FunctionsManager.ts +++ b/packages/toolpad-app/src/server/FunctionsManager.ts @@ -25,6 +25,7 @@ import type { ExtractTypesParams, IntrospectionResult } from './functionsTypesWo import { Awaitable } from '../utils/types'; import { format } from '../utils/prettier'; import { compilerOptions } from './functionsShared'; +import invariant from 'invariant'; export interface CreateDataProviderOptions { paginationMode: PaginationMode; @@ -374,7 +375,11 @@ export default class FunctionsManager { exportName: string = 'default', ): Promise { const fullPath = await this.getBuiltOutputFilePath(fileName); - return functionsRuntime.introspectDataProvider(fullPath, exportName); + const dataProvider = await functionsRuntime.loadDataProvider(fullPath, exportName); + return { + paginationMode: dataProvider.paginationMode, + hasDeleteRecord: !!dataProvider.deleteRecord, + }; } async getDataProviderRecords( @@ -383,7 +388,8 @@ export default class FunctionsManager { params: GetRecordsParams, ): Promise> { const fullPath = await this.getBuiltOutputFilePath(fileName); - return functionsRuntime.getDataProviderRecords(fullPath, exportName, params); + const dataProvider = await functionsRuntime.loadDataProvider(fullPath, exportName); + return dataProvider.getRecords(params); } async deleteDataProviderRecord( @@ -392,6 +398,8 @@ export default class FunctionsManager { id: GridRowId, ): Promise { const fullPath = await this.getBuiltOutputFilePath(fileName); - return functionsRuntime.deleteDataProviderRecord(fullPath, exportName, id); + const dataProvider = await functionsRuntime.loadDataProvider(fullPath, exportName); + invariant(dataProvider.deleteRecord, 'DataProvider does not support deleteRecord'); + return dataProvider.deleteRecord(id); } } diff --git a/packages/toolpad-app/src/server/functionsRuntime.ts b/packages/toolpad-app/src/server/functionsRuntime.ts index e6b26013dc8..3db1d4cfee7 100644 --- a/packages/toolpad-app/src/server/functionsRuntime.ts +++ b/packages/toolpad-app/src/server/functionsRuntime.ts @@ -4,13 +4,9 @@ import * as fs from 'fs/promises'; import * as vm from 'vm'; import * as url from 'node:url'; import { initialContextStore } from '@mui/toolpad-core/serverRuntime'; -import { ToolpadDataProviderIntrospection } from '@mui/toolpad-core/runtime'; import { TOOLPAD_DATA_PROVIDER_MARKER, ToolpadDataProvider } from '@mui/toolpad-core/server'; import * as z from 'zod'; import { fromZodError } from 'zod-validation-error'; -import { GetRecordsParams, GetRecordsResult, PaginationMode } from '@mui/toolpad-core'; -import invariant from 'invariant'; -import type { GridRowId } from '@mui/x-data-grid'; import.meta.url ??= url.pathToFileURL(__filename).toString(); @@ -38,7 +34,7 @@ function loadModule(fullPath: string, content: string) { return moduleObject; } -async function resolveExports(filePath: string): Promise> { +async function loadExports(filePath: string): Promise> { const fullPath = path.resolve(filePath); const content = await fs.readFile(fullPath, 'utf-8'); @@ -66,11 +62,11 @@ const dataProviderSchema: z.ZodType> = z.object({ [TOOLPAD_DATA_PROVIDER_MARKER]: z.literal(true), }); -async function loadDataProvider( +export async function loadDataProvider( filePath: string, name: string, ): Promise> { - const exports = await resolveExports(filePath); + const exports = await loadExports(filePath); const dataProviderExport = exports.get(name); if (!dataProviderExport || typeof dataProviderExport !== 'object') { @@ -86,40 +82,8 @@ async function loadDataProvider( throw fromZodError(parsed.error); } -export async function introspectDataProvider( - filePath: string, - name: string, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - - return { - paginationMode: dataProvider.paginationMode, - hasDeleteRecord: !!dataProvider.deleteRecord, - }; -} - -export async function getDataProviderRecords( - filePath: string, - name: string, - params: GetRecordsParams, -): Promise> { - const dataProvider = await loadDataProvider(filePath, name); - - return dataProvider.getRecords(params); -} - -export async function deleteDataProviderRecord( - filePath: string, - name: string, - id: GridRowId, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - invariant(dataProvider.deleteRecord, 'DataProvider does not support deleteRecord'); - return dataProvider.deleteRecord(id); -} - export async function execute(filePath: string, name: string, parameters: unknown[]): Promise { - const exports = await resolveExports(filePath); + const exports = await loadExports(filePath); const fn = exports.get(name); if (typeof fn !== 'function') { From 67086ac81decfff71ba3bd21a7d51f7b05b9a085 Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:01:40 +0100 Subject: [PATCH 5/8] Update FunctionsManager.ts --- packages/toolpad-app/src/server/FunctionsManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/toolpad-app/src/server/FunctionsManager.ts b/packages/toolpad-app/src/server/FunctionsManager.ts index eb3ace12628..c2438c3330d 100644 --- a/packages/toolpad-app/src/server/FunctionsManager.ts +++ b/packages/toolpad-app/src/server/FunctionsManager.ts @@ -18,6 +18,7 @@ import { errorFrom } from '@mui/toolpad-utils/errors'; import { ToolpadDataProviderIntrospection } from '@mui/toolpad-core/runtime'; import * as url from 'node:url'; import type { GridRowId } from '@mui/x-data-grid'; +import invariant from 'invariant'; import EnvManager from './EnvManager'; import { ProjectEvents, ToolpadProjectOptions } from '../types'; import * as functionsRuntime from './functionsRuntime'; @@ -25,7 +26,6 @@ import type { ExtractTypesParams, IntrospectionResult } from './functionsTypesWo import { Awaitable } from '../utils/types'; import { format } from '../utils/prettier'; import { compilerOptions } from './functionsShared'; -import invariant from 'invariant'; export interface CreateDataProviderOptions { paginationMode: PaginationMode; From 39933e2dfa7355fe97e5ae862eb5c19ee28a1f19 Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:57:42 +0100 Subject: [PATCH 6/8] rebase --- .../src/server/FunctionsManager.ts | 10 +- .../src/server/functionsDevWorker.ts | 279 ------------------ 2 files changed, 6 insertions(+), 283 deletions(-) delete mode 100644 packages/toolpad-app/src/server/functionsDevWorker.ts diff --git a/packages/toolpad-app/src/server/FunctionsManager.ts b/packages/toolpad-app/src/server/FunctionsManager.ts index 8cdd03a41b5..7ce769d3e6f 100644 --- a/packages/toolpad-app/src/server/FunctionsManager.ts +++ b/packages/toolpad-app/src/server/FunctionsManager.ts @@ -410,8 +410,9 @@ export default class FunctionsManager { values: Record, ): Promise { const fullPath = await this.getBuiltOutputFilePath(fileName); - invariant(this.devWorker, 'devWorker must be initialized'); - return this.devWorker.updateDataProviderRecord(fullPath, exportName, id, values); + const dataProvider = await functionsRuntime.loadDataProvider(fullPath, exportName); + invariant(dataProvider.updateRecord, 'DataProvider does not support updateRecord'); + return dataProvider.updateRecord(id, values); } async createDataProviderRecord( @@ -420,7 +421,8 @@ export default class FunctionsManager { values: Record, ): Promise { const fullPath = await this.getBuiltOutputFilePath(fileName); - invariant(this.devWorker, 'devWorker must be initialized'); - return this.devWorker.createDataProviderRecord(fullPath, exportName, values); + const dataProvider = await functionsRuntime.loadDataProvider(fullPath, exportName); + invariant(dataProvider.createRecord, 'DataProvider does not support createRecord'); + return dataProvider.createRecord(values); } } diff --git a/packages/toolpad-app/src/server/functionsDevWorker.ts b/packages/toolpad-app/src/server/functionsDevWorker.ts deleted file mode 100644 index 0ef6502e8ea..00000000000 --- a/packages/toolpad-app/src/server/functionsDevWorker.ts +++ /dev/null @@ -1,279 +0,0 @@ -import { Worker, MessageChannel, isMainThread, parentPort } from 'worker_threads'; -import * as path from 'path'; -import { createRequire } from 'node:module'; -import * as fs from 'fs/promises'; -import * as vm from 'vm'; -import * as url from 'node:url'; -import { - ServerContext, - getServerContext, - initialContextStore, - withContext, -} from '@mui/toolpad-core/serverRuntime'; -import { isWebContainer } from '@webcontainer/env'; -import * as superjson from 'superjson'; -import { createRpcClient, serveRpc } from '@mui/toolpad-utils/workerRpc'; -import { workerData } from 'node:worker_threads'; -import { ToolpadDataProviderIntrospection } from '@mui/toolpad-core/runtime'; -import { TOOLPAD_DATA_PROVIDER_MARKER, ToolpadDataProvider } from '@mui/toolpad-core/server'; -import * as z from 'zod'; -import { fromZodError } from 'zod-validation-error'; -import { GetRecordsParams, GetRecordsResult, PaginationMode } from '@mui/toolpad-core'; -import invariant from 'invariant'; -import type { GridRowId } from '@mui/x-data-grid'; - -import.meta.url ??= url.pathToFileURL(__filename).toString(); -const currentDirectory = url.fileURLToPath(new URL('.', import.meta.url)); - -interface ModuleObject { - exports: Record; -} - -const fileContents = new Map(); -const moduleCache = new Map(); - -function loadModule(fullPath: string, content: string) { - const moduleRequire = createRequire(url.pathToFileURL(fullPath)); - const moduleObject: ModuleObject = { exports: {} }; - - const toolpadServer = moduleRequire('@mui/toolpad/server'); - // eslint-disable-next-line no-underscore-dangle - toolpadServer.__initContextStore(initialContextStore); - - vm.runInThisContext(`((require, exports, module) => {\n${content}\n})`)( - moduleRequire, - moduleObject.exports, - moduleObject, - ); - - return moduleObject; -} - -async function resolveExports(filePath: string): Promise> { - const fullPath = path.resolve(filePath); - const content = await fs.readFile(fullPath, 'utf-8'); - - if (content !== fileContents.get(fullPath)) { - moduleCache.delete(fullPath); - fileContents.set(fullPath, content); - } - - let cachedModule = moduleCache.get(fullPath); - - if (!cachedModule) { - cachedModule = loadModule(fullPath, content); - moduleCache.set(fullPath, cachedModule); - } - - return new Map(Object.entries(cachedModule.exports)); -} - -interface ExecuteParams { - filePath: string; - name: string; - parameters: unknown[]; - cookies?: Record; -} - -interface ExecuteResult { - result: string; - newCookies: [string, string][]; -} - -async function execute(msg: ExecuteParams): Promise { - const exports = await resolveExports(msg.filePath); - - const fn = exports.get(msg.name); - if (typeof fn !== 'function') { - throw new Error(`Function "${msg.name}" not found`); - } - - let functionFinished = false; - - try { - const newCookies = new Map(); - - const ctx: ServerContext = { - cookies: msg.cookies || {}, - setCookie(name: string, value: string) { - if (functionFinished) { - throw new Error(`setCookie can't be called after the function has finished executing.`); - } - newCookies.set(name, value); - }, - }; - - const shouldBypassContext = isWebContainer(); - - if (shouldBypassContext) { - console.warn( - 'Bypassing server context in web containers, see https://github.com/stackblitz/core/issues/2711', - ); - } - - const rawResult = shouldBypassContext - ? await fn(...msg.parameters) - : await withContext(ctx, async () => fn(...msg.parameters)); - - const serializedResult = superjson.stringify(rawResult); - - return { result: serializedResult, newCookies: Array.from(newCookies.entries()) }; - } finally { - functionFinished = true; - } -} - -const dataProviderSchema: z.ZodType> = z.object({ - paginationMode: z.enum(['index', 'cursor']).optional().default('index'), - getRecords: z.function(z.tuple([z.any()]), z.any()), - deleteRecord: z.function(z.tuple([z.any()]), z.any()).optional(), - updateRecord: z.function(z.tuple([z.any(), z.any()]), z.any()).optional(), - createRecord: z.function(z.tuple([z.any()]), z.any()).optional(), - [TOOLPAD_DATA_PROVIDER_MARKER]: z.literal(true), -}); - -async function loadDataProvider( - filePath: string, - name: string, -): Promise> { - const exports = await resolveExports(filePath); - const dataProviderExport = exports.get(name); - - if (!dataProviderExport || typeof dataProviderExport !== 'object') { - throw new Error(`DataProvider "${name}" not found`); - } - - const parsed = dataProviderSchema.safeParse(dataProviderExport); - - if (parsed.success) { - return parsed.data; - } - - throw fromZodError(parsed.error); -} - -async function introspectDataProvider( - filePath: string, - name: string, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - - return { - paginationMode: dataProvider.paginationMode, - hasDeleteRecord: !!dataProvider.deleteRecord, - hasUpdateRecord: !!dataProvider.updateRecord, - hasCreateRecord: !!dataProvider.createRecord, - }; -} - -async function getDataProviderRecords( - filePath: string, - name: string, - params: GetRecordsParams, -): Promise> { - const dataProvider = await loadDataProvider(filePath, name); - - return dataProvider.getRecords(params); -} - -async function deleteDataProviderRecord( - filePath: string, - name: string, - id: GridRowId, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - invariant(dataProvider.deleteRecord, 'DataProvider does not support deleteRecord'); - return dataProvider.deleteRecord(id); -} - -async function updateDataProviderRecord( - filePath: string, - name: string, - id: GridRowId, - values: Record, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - invariant(dataProvider.updateRecord, 'DataProvider does not support updateRecord'); - return dataProvider.updateRecord(id, values); -} - -async function createDataProviderRecord( - filePath: string, - name: string, - values: Record, -): Promise { - const dataProvider = await loadDataProvider(filePath, name); - invariant(dataProvider.createRecord, 'DataProvider does not support createRecord'); - return dataProvider.createRecord(values); -} - -type WorkerRpcServer = { - execute: typeof execute; - introspectDataProvider: typeof introspectDataProvider; - getDataProviderRecords: typeof getDataProviderRecords; - deleteDataProviderRecord: typeof deleteDataProviderRecord; - updateDataProviderRecord: typeof updateDataProviderRecord; - createDataProviderRecord: typeof createDataProviderRecord; -}; - -if (!isMainThread && parentPort) { - serveRpc(workerData.workerRpcPort, { - execute, - introspectDataProvider, - getDataProviderRecords, - deleteDataProviderRecord, - updateDataProviderRecord, - createDataProviderRecord, - }); -} - -export function createWorker(env: Record) { - const workerRpcChannel = new MessageChannel(); - const worker = new Worker(path.resolve(currentDirectory, '../cli/functionsDevWorker.mjs'), { - env, - workerData: { - workerRpcPort: workerRpcChannel.port1, - }, - transferList: [workerRpcChannel.port1], - }); - - const client = createRpcClient(workerRpcChannel.port2); - - return { - async terminate() { - return worker.terminate(); - }, - - async execute(filePath: string, name: string, parameters: unknown[]): Promise { - const ctx = getServerContext(); - - const { result: serializedResult, newCookies } = await client.execute({ - filePath, - name, - parameters, - cookies: ctx?.cookies, - }); - - if (ctx) { - for (const [cookieName, cookieValue] of newCookies) { - ctx.setCookie(cookieName, cookieValue); - } - } - - const result = superjson.parse(serializedResult); - - return result; - }, - - introspectDataProvider: client.introspectDataProvider.bind(client), - getDataProviderRecords: client.getDataProviderRecords.bind(client), - deleteDataProviderRecord: client.deleteDataProviderRecord.bind(client), - updateDataProviderRecord: client.updateDataProviderRecord.bind(client), - createDataProviderRecord: client.createDataProviderRecord.bind(client), - }; -} - -process.on('unhandledRejection', (error) => { - console.error(error); - process.exit(1); -}); From 391e0f02fa71d03f2eb722ef8c61864c753c4fd7 Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 17:08:12 +0100 Subject: [PATCH 7/8] Update FunctionsManager.ts --- packages/toolpad-app/src/server/FunctionsManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/toolpad-app/src/server/FunctionsManager.ts b/packages/toolpad-app/src/server/FunctionsManager.ts index 7ce769d3e6f..5ca79621cba 100644 --- a/packages/toolpad-app/src/server/FunctionsManager.ts +++ b/packages/toolpad-app/src/server/FunctionsManager.ts @@ -379,6 +379,8 @@ export default class FunctionsManager { return { paginationMode: dataProvider.paginationMode, hasDeleteRecord: !!dataProvider.deleteRecord, + hasUpdateRecord: !!dataProvider.updateRecord, + hasCreateRecord: !!dataProvider.createRecord, }; } From b4f59afea816b09ddc9231ae466e92a063cc9be6 Mon Sep 17 00:00:00 2001 From: MUI bot <2109932+Janpot@users.noreply.github.com> Date: Wed, 20 Dec 2023 17:38:53 +0100 Subject: [PATCH 8/8] Update functionsRuntime.ts --- packages/toolpad-app/src/server/functionsRuntime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/toolpad-app/src/server/functionsRuntime.ts b/packages/toolpad-app/src/server/functionsRuntime.ts index 3db1d4cfee7..ddf6f8c9f73 100644 --- a/packages/toolpad-app/src/server/functionsRuntime.ts +++ b/packages/toolpad-app/src/server/functionsRuntime.ts @@ -57,7 +57,7 @@ const dataProviderSchema: z.ZodType> = z.object({ paginationMode: z.enum(['index', 'cursor']).optional().default('index'), getRecords: z.function(z.tuple([z.any()]), z.any()), deleteRecord: z.function(z.tuple([z.any()]), z.any()).optional(), - updateRecord: z.function(z.tuple([z.any()]), z.any()).optional(), + updateRecord: z.function(z.tuple([z.any(), z.any()]), z.any()).optional(), createRecord: z.function(z.tuple([z.any()]), z.any()).optional(), [TOOLPAD_DATA_PROVIDER_MARKER]: z.literal(true), });