UBERF-8052: Allow easy profiling of transactor

Signed-off-by: Andrey Sobolev <[email protected]>

.github/workflows/main.yml

@@ -232,10 +232,26 @@ jobs:
         run: |
           cd ./tests/sanity
           node ../../common/scripts/install-run-rushx.js ci
+      - name: Start profiling
+        run: |
+          cd ./tests
+          ./profile-start.sh
       - name: Run UI tests
         run: |
-          cd ./tests/sanity
+          cd ./tests/sanity          
           node ../../common/scripts/install-run-rushx.js uitest
+      - name: Download profile
+        run: |
+          cd ./tests
+          ./profile-download.sh
+          npm install -g cpupro
+          ./profile-generate.sh
+      - name: Upload profiling results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: profiling
+          path: ./tests/profiling
       - name: 'Store docker logs'
         if: always()
         run: |

.gitignore

@@ -100,3 +100,4 @@ services/github/pod-github/src/github.graphql
 dev/tool/report.csv
 bundle/*
 bundle.js.map
+tests/profiles

dev/tool/src/__start.ts

@@ -82,6 +82,4 @@ function prepareTools (): {
   return { ...prepareToolsRaw(builder(enabled, disabled).getTxes()), version: getModelVersion(), migrateOperations }
 }
 
-console.log(`tools git_version: ${process.env.GIT_REVISION ?? ''} model_version: ${process.env.MODEL_VERSION ?? ''}`)
-
 devTool(prepareTools)

dev/tool/src/index.ts

@@ -187,6 +187,12 @@ export function devTool (
 
   program.version('0.0.1')
 
+  program.command('version').action(() => {
+    console.log(
+      `tools git_version: ${process.env.GIT_REVISION ?? ''} model_version: ${process.env.MODEL_VERSION ?? ''}`
+    )
+  })
+
   // create-account [email protected] --password 123 --workspace workspace --fullname "John Appleseed"
   program
     .command('create-account <email>')
@@ -937,8 +943,9 @@ export function devTool (
   program
     .command('generate-token <name> <workspace>')
     .description('generate token')
-    .action(async (name: string, workspace: string) => {
-      console.log(generateToken(name, getWorkspaceId(workspace)))
+    .option('--admin', 'Generate token with admin access', false)
+    .action(async (name: string, workspace: string, opt: { admin: boolean }) => {
+      console.log(generateToken(name, getWorkspaceId(workspace), { ...(opt.admin ? { admin: 'true' } : {}) }))
     })
   program
     .command('decode-token <token>')

packages/core/src/server.ts

@@ -56,26 +56,6 @@ export interface SessionData {
   branding: Branding | null
 }
 
-export interface ContextData {
-  derived: {
-    txes: Tx[]
-    targets: BroadcastTargets // A set of broadcast filters if required
-  }
-  contextCache: Map<string, any>
-  removedMap: Map<Ref<Doc>, Doc>
-
-  userEmail: string
-  sessionId: string
-  admin?: boolean
-
-  account: Account
-
-  getAccount: (account: Ref<Account>) => Account | undefined
-
-  workspace: WorkspaceIdWithUrl
-  branding: Branding | null
-}
-
 /**
  * @public
  */

packages/text/src/markup/utils.ts

@@ -25,6 +25,7 @@ import { MarkupMark, MarkupNode, MarkupNodeType, emptyMarkupNode } from './model
 
 /** @public */
 export const EmptyMarkup: Markup = jsonToMarkup(emptyMarkupNode())
+const defaultSchema = getSchema(defaultExtensions)
 
 /** @public */
 export function getMarkup (editor?: Editor): Markup {
@@ -157,7 +158,7 @@ export function markupToJSON (markup: Markup): MarkupNode {
 
 /** @public */
 export function jsonToPmNode (json: MarkupNode, schema?: Schema, extensions?: Extensions): ProseMirrorNode {
-  schema ??= getSchema(extensions ?? defaultExtensions)
+  schema ??= extensions == null ? defaultSchema : getSchema(extensions ?? defaultExtensions)
   return ProseMirrorNode.fromJSON(schema, json)
 }
 
@@ -210,7 +211,7 @@ export function jsonToHTML (json: MarkupNode, extensions?: Extensions): string {
 
 /** @public */
 export function htmlToPmNode (html: string, schema?: Schema, extensions?: Extensions): ProseMirrorNode {
-  schema ??= getSchema(extensions ?? defaultExtensions)
+  schema ??= extensions == null ? defaultSchema : getSchema(extensions ?? defaultExtensions)
   const json = htmlToJSON(html, extensions)
   return ProseMirrorNode.fromJSON(schema, json)
 }
@@ -226,8 +227,6 @@ export function pmNodeToHTML (node: ProseMirrorNode, extensions?: Extensions): s
 const ELLIPSIS_CHAR = '…'
 const WHITESPACE = ' '
 
-const defaultSchema = getSchema(defaultExtensions)
-
 /** @public */
 export function stripTags (markup: Markup, textLimit = 0, extensions: Extensions | undefined = undefined): string {
   const schema = extensions === undefined ? defaultSchema : getSchema(extensions)

plugins/workbench-resources/src/components/ServerManagerGeneral.svelte

@@ -3,7 +3,7 @@
   import login from '@hcengineering/login'
   import { getEmbeddedLabel, getMetadata } from '@hcengineering/platform'
   import presentation, { getClient, isAdminUser } from '@hcengineering/presentation'
-  import { Button, IconArrowLeft, IconArrowRight, fetchMetadataLocalStorage } from '@hcengineering/ui'
+  import { Button, IconArrowLeft, IconArrowRight, fetchMetadataLocalStorage, ticker } from '@hcengineering/ui'
   import EditBox from '@hcengineering/ui/src/components/EditBox.svelte'
 
   const _endpoint: string = fetchMetadataLocalStorage(login.metadata.LoginEndpoint) ?? ''
@@ -34,6 +34,22 @@
 
   let responseSize = 0
 
+  let profiling = false
+
+  async function fetchStats (time: number): Promise<void> {
+    await fetch(endpoint + `/api/v1/profiling?token=${token}`, {})
+      .then(async (json) => {
+        data = await json.json()
+        profiling = data?.profiling ?? false
+      })
+      .catch((err) => {
+        console.error(err)
+      })
+  }
+  let data: any
+
+  $: void fetchStats($ticker)
+
   function genData (dataSize: number): string {
     let result = ''
     const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
@@ -97,6 +113,26 @@
     running = false
     clearInterval(int)
   }
+
+  async function downloadProfile (): Promise<void> {
+    const link = document.createElement('a')
+    link.style.display = 'none'
+    link.setAttribute('target', '_blank')
+    const json = await (
+      await fetch(endpoint + `/api/v1/manage?token=${token}&operation=profile-stop`, {
+        method: 'PUT'
+      })
+    ).json()
+    link.setAttribute(
+      'href',
+      'data:application/json;charset=utf-8,%EF%BB%BF' + encodeURIComponent(JSON.stringify(json))
+    )
+    link.setAttribute('download', `profile-${Date.now()}.cpuprofile`)
+    document.body.appendChild(link)
+    link.click()
+    document.body.removeChild(link)
+    fetchStats(0)
+  }
 </script>
 
 {#if isAdminUser()}
@@ -176,6 +212,22 @@
         }}
       />
     </div>
+    <div class="flex-row-center p-1">
+      <div class="p-3">3.</div>
+      {#if !profiling}
+        <Button
+          label={getEmbeddedLabel('Profile server')}
+          on:click={() => {
+            void fetch(endpoint + `/api/v1/manage?token=${token}&operation=profile-start`, {
+              method: 'PUT'
+            })
+            fetchStats(0)
+          }}
+        />
+      {:else}
+        <Button label={getEmbeddedLabel('Profile Stop')} on:click={downloadProfile} />
+      {/if}
+    </div>
   </div>
 {/if}
 

pods/server/src/__start.ts

@@ -19,6 +19,7 @@ import serverTelegram from '@hcengineering/server-telegram'
 import serverAiBot from '@hcengineering/server-ai-bot'
 import { join } from 'path'
 import { start } from '.'
+import { profileStart, profileStop } from './inspector'
 const serverFactory = serverFactories[(process.env.SERVER_PROVIDER as string) ?? 'ws'] ?? serverFactories.ws
 
 configureAnalytics(process.env.SENTRY_DSN, {})
@@ -68,7 +69,11 @@ const shutdown = start(config.url, {
   indexProcessing: 500,
   brandingMap: loadBrandingMap(config.brandingPath),
   accountsUrl: config.accountsUrl,
-  enableCompression: config.enableCompression
+  enableCompression: config.enableCompression,
+  profiling: {
+    start: profileStart,
+    stop: profileStop
+  }
 })
 
 const close = (): void => {

pods/server/src/inspector.ts

@@ -0,0 +1,38 @@
+import { PlatformError, unknownError } from '@hcengineering/platform'
+import * as inspector from 'node:inspector'
+
+let session: inspector.Session | undefined
+export function profileStart (): void {
+  try {
+    session = new inspector.Session()
+    session.connect()
+
+    session.post('Profiler.enable')
+    session.post('Profiler.start')
+  } catch (err) {
+    console.error(err)
+  }
+}
+
+export async function profileStop (): Promise<string | undefined> {
+  return await new Promise<string | undefined>((resolve, reject) => {
+    if (session == null) {
+      reject(new PlatformError(unknownError('no session started')))
+      return
+    }
+    try {
+      session.post('Profiler.stop', (err, profile) => {
+        if (err != null) {
+          reject(err)
+        } else {
+          const json = JSON.stringify(profile.profile)
+          session?.disconnect()
+          session = undefined
+          resolve(json)
+        }
+      })
+    } catch (err) {
+      reject(err)
+    }
+  })
+}

pods/server/src/server.ts

@@ -46,6 +46,11 @@ export function start (
     enableCompression?: boolean
 
     accountsUrl: string
+
+    profiling?: {
+      start: () => void
+      stop: () => Promise<string | undefined>
+    }
   }
 ): () => Promise<void> {
   const metrics = getMetricsContext()
@@ -88,7 +93,8 @@ export function start (
     serverFactory: opt.serverFactory,
     enableCompression: opt.enableCompression,
     accountsUrl: opt.accountsUrl,
-    externalStorage
+    externalStorage,
+    profiling: opt.profiling
   })
   return async () => {
     await externalStorage.close()

server/ws/src/blobs.ts

@@ -21,7 +21,7 @@ export async function getFile (
   file: string,
   res: BlobResponse
 ): Promise<void> {
-  const stat = await ctx.with('stat', {}, async () => await client.stat(ctx, workspace, file))
+  const stat = await ctx.with('stat', {}, () => client.stat(ctx, workspace, file))
   if (stat === undefined) {
     ctx.error('No such key', { file })
     res.cork(() => {
@@ -36,7 +36,7 @@ export async function getFile (
     { contentType: stat.contentType },
     async (ctx) => {
       try {
-        const dataStream = await ctx.with('readable', {}, async () => await client.get(ctx, workspace, file))
+        const dataStream = await ctx.with('readable', {}, () => client.get(ctx, workspace, file))
         await new Promise<void>((resolve, reject) => {
           res.cork(() => {
             res.writeHead(200, {
@@ -99,7 +99,7 @@ export async function getFileRange (
   uuid: string,
   res: BlobResponse
 ): Promise<void> {
-  const stat = await ctx.with('stats', {}, async () => await client.stat(ctx, workspace, uuid))
+  const stat = await ctx.with('stats', {}, () => client.stat(ctx, workspace, uuid))
   if (stat === undefined) {
     ctx.error('No such key', { file: uuid })
     res.cork(() => {
@@ -133,7 +133,7 @@ export async function getFileRange (
         const dataStream = await ctx.with(
           'partial',
           {},
-          async () => await client.partial(ctx, workspace, uuid, start, end - start + 1),
+          () => client.partial(ctx, workspace, uuid, start, end - start + 1),
           {}
         )
         await new Promise<void>((resolve, reject) => {

server/ws/src/client.ts

@@ -108,11 +108,7 @@ export class ClientSession implements Session {
       this._pipeline.context.modelDb
     )
     ctx.ctx.contextData = contextData
-    const result = await ctx.ctx.with(
-      'load-model',
-      {},
-      async () => await this._pipeline.loadModel(ctx.ctx, lastModelTx, hash)
-    )
+    const result = await ctx.ctx.with('load-model', {}, () => this._pipeline.loadModel(ctx.ctx, lastModelTx, hash))
     await ctx.sendResponse(result)
   }
 
@@ -265,7 +261,7 @@ export class ClientSession implements Session {
       }
       const bevent = createBroadcastEvent(Array.from(classes))
       this.broadcastTx = []
-      await socket.send(
+      socket.send(
         ctx,
         {
           result: [bevent]