fix: support pg tracing in dd-trace-js (#10424)

Signed-off-by: Matt Krick <[email protected]>

.gitignore

@@ -15,7 +15,7 @@ dist/
 dump.rdb
 node_modules/
 package-lock.json
-packages/server/postgres/pg.d.ts
+packages/server/postgres/types/pg.d.ts
 packages/server/postgres/queries/generated
 packages/server/types/githubTypes.ts
 packages/server/types/gitlabTypes.ts

package.json

@@ -27,7 +27,7 @@
     "clean": "git clean -fdx -e .env",
     "codegen": "node scripts/codegenGraphQL.js",
     "pg:build": "pgtyped -c ./packages/server/postgres/pgtypedConfig.js",
-    "pg:generate": "export $(grep ^POSTGRES_ .env | tr -d \"'\"); yarn kysely-codegen --exclude-pattern \"(PgMigrations|StripeQuantityMismatchLogging)\" --out-file ./packages/server/postgres/pg.d.ts --dialect postgres --url postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB && prettier --write ./packages/server/postgres/pg.d.ts",
+    "pg:generate": "export $(grep ^POSTGRES_ .env | tr -d \"'\"); yarn kysely-codegen --exclude-pattern \"(PgMigrations|StripeQuantityMismatchLogging)\" --out-file ./packages/server/postgres/types/pg.d.ts --dialect postgres --url postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB && prettier --write ./packages/server/postgres/types/pg.d.ts",
     "pg:restore": "node ./scripts/toolbox/pgRestore.js",
     "db:start": "docker compose -f docker/stacks/development/docker-compose.yml up -d",
     "db:stop": "docker compose -f docker/stacks/development/docker-compose.yml down",

packages/embedder/addEmbeddingsMetadataForMeetingTemplate.ts

@@ -1,5 +1,5 @@
 import {ExpressionOrFactory, SqlBool, sql} from 'kysely'
-import {DB} from 'parabol-server/postgres/pg'
+import {DB} from 'parabol-server/postgres/types/pg'
 import {Logger} from 'parabol-server/utils/Logger'
 import getKysely from '../server/postgres/getKysely'
 import {AddEmbeddingsMetadataParams} from './addEmbeddingsMetadata'

packages/embedder/addEmbeddingsMetadataForRetrospectiveDiscussionTopic.ts

@@ -1,5 +1,5 @@
 import {ExpressionOrFactory, SqlBool, sql} from 'kysely'
-import {DB} from 'parabol-server/postgres/pg'
+import {DB} from 'parabol-server/postgres/types/pg'
 import {Logger} from 'parabol-server/utils/Logger'
 import getKysely from '../server/postgres/getKysely'
 import {AddEmbeddingsMetadataParams} from './addEmbeddingsMetadata'

packages/embedder/ai_models/AbstractEmbeddingsModel.ts

@@ -1,6 +1,6 @@
 import {sql} from 'kysely'
 import getKysely from 'parabol-server/postgres/getKysely'
-import {DB} from 'parabol-server/postgres/pg'
+import {DB} from 'parabol-server/postgres/types/pg'
 import isValid from '../../server/graphql/isValid'
 import {Logger} from '../../server/utils/Logger'
 import {getEmbedderPriority} from '../getEmbedderPriority'

packages/embedder/custom.d.ts

@@ -1,5 +1,5 @@
 import {DataLoaderInstance} from '../server/dataloader/RootDataLoader'
-import type {DB} from '../server/postgres/pg'
+import type {DB} from '../server/postgres/types/pg'
 import {JobQueueError} from './JobQueueError'
 
 export type EmbeddingObjectType = DB['EmbeddingsMetadata']['objectType']

packages/embedder/indexing/createEmbeddingTextFrom.ts

@@ -1,5 +1,5 @@
 import {Selectable} from 'kysely'
-import {DB} from 'parabol-server/postgres/pg'
+import {DB} from 'parabol-server/postgres/types/pg'
 
 import {DataLoaderInstance} from '../../server/dataloader/RootDataLoader'
 import {createTextFromMeetingTemplate} from './meetingTemplate'

packages/server/billing/helpers/adjustUserCount.ts

@@ -4,9 +4,9 @@ import generateUID from '../../generateUID'
 import {DataLoaderWorker} from '../../graphql/graphql'
 import isValid from '../../graphql/isValid'
 import getKysely from '../../postgres/getKysely'
-import {OrganizationUserAudit} from '../../postgres/pg'
 import {getUserById} from '../../postgres/queries/getUsersByIds'
 import IUser from '../../postgres/types/IUser'
+import {OrganizationUserAudit} from '../../postgres/types/pg'
 import {Logger} from '../../utils/Logger'
 import {analytics} from '../../utils/analytics/analytics'
 import getActiveDomainForOrgId from '../../utils/getActiveDomainForOrgId'

packages/server/database/types/MeetingTemplate.ts

@@ -1,7 +1,7 @@
 import {Insertable} from 'kysely'
 import generateUID from '../../generateUID'
-import {MeetingTemplate as MeetingTemplateDB} from '../../postgres/pg'
 import {MeetingTypeEnum} from '../../postgres/types/Meeting'
+import {MeetingTemplate as MeetingTemplateDB} from '../../postgres/types/pg'
 
 export type SharingScopeEnum = 'ORGANIZATION' | 'PUBLIC' | 'TEAM' | 'USER'
 

packages/server/dataloader/__tests__/isOrgVerified.test.ts

@@ -3,8 +3,8 @@ import {Insertable} from 'kysely'
 import {createPGTables, truncatePGTables} from '../../__tests__/common'
 import generateUID from '../../generateUID'
 import getKysely from '../../postgres/getKysely'
-import {User} from '../../postgres/pg'
 import {OrganizationUser} from '../../postgres/types'
+import {User} from '../../postgres/types/pg'
 import RootDataLoader from '../RootDataLoader'
 
 const TEST_DB = 'getVerifiedOrgIdsTest'

packages/server/dataloader/customLoaderMakers.ts

@@ -6,7 +6,6 @@ import getFileStoreManager from '../fileStorage/getFileStoreManager'
 import {ReactableEnum} from '../graphql/public/resolverTypes'
 import {SAMLSource} from '../graphql/public/types/SAML'
 import getKysely from '../postgres/getKysely'
-import {TeamMeetingTemplate} from '../postgres/pg.d'
 import {IGetLatestTaskEstimatesQueryResult} from '../postgres/queries/generated/getLatestTaskEstimatesQuery'
 import getGitHubAuthByUserIdTeamId, {
   GitHubAuth
@@ -29,6 +28,7 @@ import {
 } from '../postgres/select'
 import {Insight, MeetingSettings, OrganizationUser, Task, Team} from '../postgres/types'
 import {AnyMeeting, MeetingTypeEnum} from '../postgres/types/Meeting'
+import {TeamMeetingTemplate} from '../postgres/types/pg'
 import {Logger} from '../utils/Logger'
 import getRedis from '../utils/getRedis'
 import isUserVerified from '../utils/isUserVerified'

packages/server/dataloader/integrationAuthLoaders.ts

@@ -3,7 +3,6 @@ import TeamMemberIntegrationAuthId from '../../client/shared/gqlIds/TeamMemberIn
 import errorFilter from '../graphql/errorFilter'
 import isValid from '../graphql/isValid'
 import getKysely from '../postgres/getKysely'
-import {TeamMemberIntegrationAuth} from '../postgres/pg'
 import {IGetBestTeamIntegrationAuthQueryResult} from '../postgres/queries/generated/getBestTeamIntegrationAuthQuery'
 import {IntegrationProviderServiceEnum} from '../postgres/queries/generated/getIntegrationProvidersByIdsQuery'
 import {IGetTeamMemberIntegrationAuthQueryResult} from '../postgres/queries/generated/getTeamMemberIntegrationAuthQuery'
@@ -14,6 +13,7 @@ import getIntegrationProvidersByIds, {
 import getTeamMemberIntegrationAuth from '../postgres/queries/getTeamMemberIntegrationAuth'
 import {selectSlackNotifications} from '../postgres/select'
 import {SlackAuth, SlackNotification} from '../postgres/types'
+import {TeamMemberIntegrationAuth} from '../postgres/types/pg'
 import NullableDataLoader from './NullableDataLoader'
 import RootDataLoader from './RootDataLoader'
 

packages/server/graphql/mutations/createTask.ts

@@ -11,8 +11,8 @@ import getTagsFromEntityMap from '../../../client/utils/draftjs/getTagsFromEntit
 import generateUID from '../../generateUID'
 import updatePrevUsedRepoIntegrationsCache from '../../integrations/updatePrevUsedRepoIntegrationsCache'
 import getKysely from '../../postgres/getKysely'
-import {Notification} from '../../postgres/pg'
 import {Task, TaskTag} from '../../postgres/types/index.d'
+import {Notification} from '../../postgres/types/pg'
 import {TaskServiceEnum} from '../../postgres/types/TaskIntegration'
 import {analytics} from '../../utils/analytics/analytics'
 import {getUserId, isTeamMember} from '../../utils/authorization'

packages/server/graphql/mutations/helpers/addAIGeneratedContentToThreads.ts

@@ -4,7 +4,7 @@ import extractTextFromDraftString from '../../../../client/utils/draftjs/extract
 import DiscussStage from '../../../database/types/DiscussStage'
 import generateUID from '../../../generateUID'
 import getKysely from '../../../postgres/getKysely'
-import {Comment} from '../../../postgres/pg'
+import {Comment} from '../../../postgres/types/pg'
 import {convertHtmlToTaskContent} from '../../../utils/draftjs/convertHtmlToTaskContent'
 import {DataLoaderWorker} from '../../graphql'
 

packages/server/graphql/mutations/helpers/createNewMeetingPhases.ts

@@ -24,9 +24,9 @@ import UpdatesPhase from '../../../database/types/UpdatesPhase'
 import UpdatesStage from '../../../database/types/UpdatesStage'
 import {DataLoaderInstance} from '../../../dataloader/RootDataLoader'
 import getKysely from '../../../postgres/getKysely'
-import {DB} from '../../../postgres/pg'
 import {MeetingTypeEnum} from '../../../postgres/types/Meeting'
 import {NewMeetingPhase, NewMeetingStages} from '../../../postgres/types/NewMeetingPhase'
+import {DB} from '../../../postgres/types/pg'
 import isPhaseAvailable from '../../../utils/isPhaseAvailable'
 import {DataLoaderWorker} from '../../graphql'
 import {getFeatureTier} from '../../types/helpers/getFeatureTier'

packages/server/graphql/mutations/helpers/notifications/MattermostNotifier.ts

@@ -4,11 +4,11 @@ import makeAppURL from 'parabol-client/utils/makeAppURL'
 import findStageById from 'parabol-client/utils/meetings/findStageById'
 import {phaseLabelLookup} from 'parabol-client/utils/meetings/lookups'
 import appOrigin from '../../../../appOrigin'
-import {TeamMemberIntegrationAuth} from '../../../../postgres/pg'
 import {IntegrationProviderMattermost} from '../../../../postgres/queries/getIntegrationProvidersByIds'
 import {SlackNotification, Team} from '../../../../postgres/types'
 import IUser from '../../../../postgres/types/IUser'
 import {AnyMeeting, MeetingTypeEnum} from '../../../../postgres/types/Meeting'
+import {TeamMemberIntegrationAuth} from '../../../../postgres/types/pg'
 import MattermostServerManager from '../../../../utils/MattermostServerManager'
 import {analytics} from '../../../../utils/analytics/analytics'
 import {toEpochSeconds} from '../../../../utils/epochTime'

packages/server/graphql/mutations/helpers/removeScheduledJobs.ts

@@ -1,6 +1,6 @@
 import {Updateable} from 'kysely'
 import getKysely from '../../../postgres/getKysely'
-import {DB} from '../../../postgres/pg'
+import {DB} from '../../../postgres/types/pg'
 
 type FilterType = Omit<Updateable<DB['ScheduledJob']>, 'runAt'>
 

packages/server/graphql/mutations/joinMeeting.ts

@@ -7,10 +7,10 @@ import CheckInStage from '../../database/types/CheckInStage'
 import TeamPromptResponseStage from '../../database/types/TeamPromptResponseStage'
 import UpdatesStage from '../../database/types/UpdatesStage'
 import getKysely from '../../postgres/getKysely'
-import {MeetingMember} from '../../postgres/pg'
 import {TeamMember} from '../../postgres/types'
 import {AnyMeeting} from '../../postgres/types/Meeting'
 import {NewMeetingPhase, NewMeetingStages} from '../../postgres/types/NewMeetingPhase'
+import {MeetingMember} from '../../postgres/types/pg'
 import {analytics} from '../../utils/analytics/analytics'
 import {getUserId, isTeamMember} from '../../utils/authorization'
 import getPhase from '../../utils/getPhase'

packages/server/graphql/mutations/updatePokerScope.ts

@@ -4,13 +4,13 @@ import {SubscriptionChannel, Threshold} from 'parabol-client/types/constEnums'
 import {ESTIMATE_TASK_SORT_ORDER} from '../../../client/utils/constants'
 import EstimateStage from '../../database/types/EstimateStage'
 import getKysely from '../../postgres/getKysely'
-import {Discussion} from '../../postgres/pg'
+import {Discussion} from '../../postgres/types/pg'
 import {TaskServiceEnum} from '../../postgres/types/TaskIntegration'
-import RedisLockQueue from '../../utils/RedisLockQueue'
 import {getUserId, isTeamMember} from '../../utils/authorization'
 import getPhase from '../../utils/getPhase'
 import getRedis from '../../utils/getRedis'
 import publish from '../../utils/publish'
+import RedisLockQueue from '../../utils/RedisLockQueue'
 import {GQLContext} from '../graphql'
 import UpdatePokerScopeItemInput from '../types/UpdatePokerScopeItemInput'
 import UpdatePokerScopePayload from '../types/UpdatePokerScopePayload'

packages/server/graphql/private/mutations/runScheduledJobs.ts

@@ -5,7 +5,7 @@ import ScheduledTeamLimitsJob from '../../../database/types/ScheduledTeamLimitsJ
 import processTeamsLimitsJob from '../../../database/types/processTeamsLimitsJob'
 import generateUID from '../../../generateUID'
 import getKysely from '../../../postgres/getKysely'
-import {DB} from '../../../postgres/pg'
+import {DB} from '../../../postgres/types/pg'
 import {Logger} from '../../../utils/Logger'
 import publish from '../../../utils/publish'
 import {DataLoaderWorker} from '../../graphql'

packages/server/graphql/public/types/RetroReflectionGroup.ts

@@ -1,5 +1,5 @@
 import {Selectable} from 'kysely'
-import {RetroReflectionGroup as TRetroReflectionGroup} from '../../../postgres/pg'
+import {RetroReflectionGroup as TRetroReflectionGroup} from '../../../postgres/types/pg'
 import {getUserId} from '../../../utils/authorization'
 import {RetroReflectionGroupResolvers} from '../resolverTypes'
 

packages/server/graphql/public/types/SAML.ts

@@ -1,5 +1,5 @@
 import {Selectable} from 'kysely'
-import {SAML as TSAML} from '../../../postgres/pg.d'
+import {SAML as TSAML} from '../../../postgres/types/pg'
 import {SamlResolvers} from '../resolverTypes'
 
 export interface SAMLSource extends Selectable<TSAML> {

packages/server/postgres/getKysely.ts

@@ -1,6 +1,6 @@
 import {Kysely, PostgresDialect} from 'kysely'
 import getPg from './getPg'
-import {DB} from './pg.d'
+import {DB} from './types/pg'
 
 let kysely: Kysely<DB> | undefined
 

packages/server/postgres/getPg.ts

@@ -1,7 +1,17 @@
-import {Pool} from 'pg'
+import type {Pool as PGPool} from 'pg'
 import sleep from '../../client/utils/sleep'
 import getPgConfig from './getPgConfig'
+/*
+Force a native `require` so dd-trace-js can monkeypatch the require statement.
+In development, the require statement requires `./pg.ts` since require resolves packages by first looking in the same dir
+In production, the require statement will resolve to the node_modules found in /dist
+*/
 
+// when used outside of webpack, e.g. kysely.config.js, go vanilla
+declare let __non_webpack_require__: typeof require
+const pg =
+  typeof __non_webpack_require__ === 'undefined' ? require('pg') : __non_webpack_require__('pg')
+const {Pool} = pg
 const config = getPgConfig()
 
 const graceFullyReconnect = async () => {
@@ -21,10 +31,10 @@ const graceFullyReconnect = async () => {
   }
 }
 
-let pool: Pool | undefined
+let pool: PGPool | undefined
 const getPg = (schema?: string) => {
   if (!pool) {
-    pool = new Pool(config)
+    pool = new Pool(config) as PGPool
     pool.on('error', graceFullyReconnect)
     if (schema) {
       pool.on('connect', (client) => {

packages/server/postgres/pg.ts

@@ -0,0 +1,2 @@
+// webpack entrypoint forcing pg & all dependencies to get bundled into a standalone file
+export {Client, Pool} from 'pg'

packages/server/postgres/types/Meeting.d.ts

@@ -1,6 +1,5 @@
 import {Selectable} from 'kysely'
 import {NonNullableProps} from '../../../client/types/generics'
-import {NewMeeting as NewMeetingPG} from '../pg.d'
 import {AutogroupReflectionGroupType, UsedReactjis} from './index.d'
 import {
   CheckInMeetingPhase,
@@ -9,6 +8,7 @@ import {
   RetroMeetingPhase,
   TeamPromptPhase
 } from './NewMeetingPhase'
+import {NewMeeting as NewMeetingPG} from './pg'
 
 type NewMeeting = Selectable<NewMeetingPG>
 export type MeetingTypeEnum = NewMeeting['meetingType']

packages/server/postgres/types/Notification.d.ts

@@ -1,4 +1,4 @@
-import type {Notification} from '../pg.d'
+import type {Notification} from './pg'
 
 interface BaseNotification {
   id: string

packages/server/postgres/types/index.d.ts

@@ -1,11 +1,4 @@
 import {SelectQueryBuilder, Selectable} from 'kysely'
-import {
-  Discussion as DiscussionPG,
-  Insight as InsightPG,
-  OrganizationUser as OrganizationUserPG,
-  TaskEstimate as TaskEstimatePG,
-  TeamMember as TeamMemberPG
-} from '../pg.d'
 import {
   selectAgendaItems,
   selectComments,
@@ -25,6 +18,13 @@ import {
   selectTemplateScale,
   selectTemplateScaleRef
 } from '../select'
+import {
+  Discussion as DiscussionPG,
+  Insight as InsightPG,
+  OrganizationUser as OrganizationUserPG,
+  TaskEstimate as TaskEstimatePG,
+  TeamMember as TeamMemberPG
+} from './pg'
 
 type ExtractTypeFromQueryBuilderSelect<T extends (...args: any[]) => any> =
   ReturnType<T> extends SelectQueryBuilder<_, _, infer X> ? X : never

packages/server/safeMutations/removeSuggestedAction.ts

@@ -1,6 +1,6 @@
 import {sql} from 'kysely'
 import getKysely from '../postgres/getKysely'
-import {SuggestedAction} from '../postgres/pg'
+import {SuggestedAction} from '../postgres/types/pg'
 
 const removeSuggestedAction = async (userId: string, type: SuggestedAction['type']) => {
   const removedAction = await getKysely()

packages/server/utils/__tests__/isRequestToJoinDomainAllowed.test.ts

@@ -5,8 +5,8 @@ import RootDataLoader from '../../dataloader/RootDataLoader'
 import generateUID from '../../generateUID'
 import {TierEnum} from '../../graphql/public/resolverTypes'
 import getKysely from '../../postgres/getKysely'
-import {User} from '../../postgres/pg'
 import {OrganizationUser} from '../../postgres/types'
+import {User} from '../../postgres/types/pg'
 import getRedis from '../getRedis'
 import {getEligibleOrgIdsByDomain} from '../isRequestToJoinDomainAllowed'
 

scripts/prod.js

@@ -13,6 +13,8 @@ const runChild = (cmd) => {
       }
     })
     build.stderr.pipe(process.stderr)
+    // enable this for debugging webpack scripts
+    // build.stdout.pipe(process.stdout)
   })
 }
 

scripts/webpack/prod.servers.config.js

@@ -44,10 +44,21 @@ module.exports = (config) => {
       ],
       pushToCDN: [DOTENV, path.join(PROJECT_ROOT, 'scripts/toolboxSrc/pushToCDN.ts')],
       migrate: [DOTENV, path.join(PROJECT_ROOT, 'scripts/toolboxSrc/standaloneMigrations.ts')],
-      assignSURole: [DOTENV, path.join(PROJECT_ROOT, 'scripts/toolboxSrc/assignSURole.ts')]
+      assignSURole: [DOTENV, path.join(PROJECT_ROOT, 'scripts/toolboxSrc/assignSURole.ts')],
+      pg: {
+        // bundle pg with all its dependencies into a single file
+        // so dd-trace-js can monkeypatch require('pg')
+        import: [path.join(SERVER_ROOT, 'postgres/pg.ts')],
+        library: {
+          type: 'commonjs2'
+        }
+      }
     },
     output: {
-      filename: '[name].js',
+      filename: (pathData) => {
+        // trick dd-trace-js into thinking our standalone pg file is a node_module
+        return pathData.chunk.name === 'pg' ? 'node_modules/pg/lib/index.js' : '[name].js'
+      },
       path: distPath
     },
     resolve: {
@@ -104,6 +115,12 @@ module.exports = (config) => {
               // copy sharp's libvips to the output
               from: path.resolve(PROJECT_ROOT, 'node_modules', 'sharp', 'vendor'),
               to: 'vendor'
+            },
+            {
+              // dd-trace-js has a lookup table for hooks, which includes the key `pg`
+              // In order for `pg` to get parsed as `pg` and not `pg.js`, we need a package.json to provide the name `pg`
+              from: path.resolve(PROJECT_ROOT, 'node_modules', 'pg', 'package.json'),
+              to: 'node_modules/pg/package.json'
             }
           ]
         })