-
Notifications
You must be signed in to change notification settings - Fork 105
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: replace DB polling with postgres notification listener for migr…
…ations - Adds HasuraClient to encapsulate behaviour, keeping the Db free from Hasura concerns. - Moves Cardano DB Hasura config into package - Adds config to nixos service - cardano-db-sync now considers EBBs as part of the epoch blocksCount, so test assertions updated
- Loading branch information
Showing
51 changed files
with
1,608 additions
and
214 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
|
|
||
| export interface Config { | ||
| db: { | ||
| database: string, | ||
| host: string, | ||
| password: string, | ||
| port: number | ||
| user: string, | ||
| }, | ||
| hasuraUri: string | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,113 +1,48 @@ | ||
| import { ApolloClient, gql, InMemoryCache, NormalizedCacheObject } from 'apollo-boost' | ||
| import { createHttpLink } from 'apollo-link-http' | ||
| import fetch from 'cross-fetch' | ||
| import utc from 'dayjs/plugin/utc' | ||
| import dayjs from 'dayjs' | ||
| import { exec } from 'child_process' | ||
| import { clearIntervalAsync, setIntervalAsync, SetIntervalAsyncTimer } from 'set-interval-async/dynamic' | ||
| import path from 'path' | ||
| import pRetry from 'p-retry' | ||
| import util from '@cardano-graphql/util' | ||
|
|
||
| dayjs.extend(utc) | ||
| import { ClientConfig } from 'pg' | ||
| import createSubscriber, { Subscriber } from 'pg-listen' | ||
|
|
||
| export class Db { | ||
| hasuraClient: ApolloClient<NormalizedCacheObject> | ||
| hasuraUri: string | ||
| monitorTimer: SetIntervalAsyncTimer | ||
| pgSubscriber: Subscriber | ||
|
|
||
| constructor (hasuraUri: string) { | ||
| this.hasuraUri = hasuraUri | ||
| this.hasuraClient = new ApolloClient({ | ||
| cache: new InMemoryCache({ | ||
| addTypename: false | ||
| }), | ||
| defaultOptions: { | ||
| query: { | ||
| fetchPolicy: 'network-only' | ||
| } | ||
| }, | ||
| link: createHttpLink({ | ||
| uri: `${this.hasuraUri}/v1/graphql`, | ||
| fetch, | ||
| headers: { | ||
| 'X-Hasura-Role': 'cardano-graphql' | ||
| } | ||
| }) | ||
| constructor (pgClientConfig: ClientConfig) { | ||
| this.pgSubscriber = createSubscriber(pgClientConfig, { | ||
| parse: (value) => value | ||
| }) | ||
| } | ||
|
|
||
| public async init (): Promise<void> { | ||
| // Todo: optimal solution dependent on https://github.com/input-output-hk/cardano-db-sync/issues/182 | ||
| await this.applySchemaAndMetadata() | ||
| this.monitorDbState() | ||
| } | ||
|
|
||
| public async shutdown () { | ||
| await clearIntervalAsync(this.monitorTimer) | ||
| } | ||
|
|
||
| private monitorDbState () { | ||
| this.monitorTimer = setIntervalAsync( | ||
| async () => { | ||
| try { | ||
| await this.getMeta() | ||
| } catch (error) { | ||
| if (error.message === 'GraphQL error: field "cardano" not found in type: \'query_root\'' || error.message === 'GraphQL error: database query error') { | ||
| console.warn('Re-applying PostgreSQL migrations and Hasura metadata') | ||
| await this.applySchemaAndMetadata() | ||
| } else { | ||
| console.error(error) | ||
| } | ||
| } | ||
| }, | ||
| 10000 | ||
| ) | ||
| } | ||
|
|
||
| async getMeta () { | ||
| const result = await this.hasuraClient.query({ | ||
| query: gql`query { | ||
| cardano { | ||
| tip { | ||
| forgedAt | ||
| } | ||
| }}` | ||
| public async init ({ onDbSetup }: { onDbSetup: Function }): Promise<void> { | ||
| this.pgSubscriber.events.on('connected', async () => { | ||
| console.log('DbClient.pgSubscriber: Connected') | ||
| await onDbSetup() | ||
| }) | ||
| const { tip } = result.data?.cardano[0] | ||
| const currentUtc = dayjs().utc() | ||
| const tipUtc = dayjs.utc(tip.forgedAt) | ||
| return { | ||
| initialized: tipUtc.isAfter(currentUtc.subtract(120, 'second')), | ||
| syncPercentage: (tipUtc.valueOf() / currentUtc.valueOf()) * 100 | ||
| } | ||
| } | ||
|
|
||
| public async applySchemaAndMetadata () { | ||
| await pRetry(async () => { | ||
| await this.hasuraCli('migrate apply --down all') | ||
| await this.hasuraCli('migrate apply --up all') | ||
| await this.hasuraCli('metadata clear') | ||
| await this.hasuraCli('metadata apply') | ||
| }, { | ||
| factor: 1.75, | ||
| retries: 9, | ||
| onFailedAttempt: util.onFailedAttemptFor('Applying PostgreSQL schema and Hasura metadata') | ||
| this.pgSubscriber.events.on('reconnect', (attempt) => { | ||
| console.warn(`DbClient.pgSubscriber: Reconnecting attempt ${attempt}`) | ||
| }) | ||
| this.pgSubscriber.events.on('error', (error) => { | ||
| console.error('DbClient.pgSubscriber: Fatal database connection error:', error) | ||
| process.exit(1) | ||
| }) | ||
| this.pgSubscriber.notifications.on('cardano_db_sync_startup', async payload => { | ||
| switch (payload) { | ||
| case 'init' : | ||
| console.log('DbClient.pgSubscriber: cardano-db-sync-extended starting, schema will be reset') | ||
| break | ||
| case 'db-setup' : | ||
| await onDbSetup() | ||
| break | ||
| default : | ||
| console.error(`DbClient.pgSubscriber: Unknown message payload ${payload}`) | ||
| } | ||
| }) | ||
| try { | ||
| await this.pgSubscriber.connect() | ||
| await this.pgSubscriber.listenTo('cardano_db_sync_startup') | ||
| } catch (error) { | ||
| console.error(error) | ||
| } | ||
| } | ||
|
|
||
| private hasuraCli (command: string) { | ||
| return new Promise((resolve, reject) => { | ||
| exec( | ||
| `hasura --skip-update-check --project ${path.resolve(__dirname, '..', 'hasura', 'project')} --endpoint ${this.hasuraUri} ${command}`, | ||
| (error, stdout) => { | ||
| if (error) { | ||
| reject(error) | ||
| } | ||
| console.log(stdout) | ||
| resolve() | ||
| } | ||
| ) | ||
| }) | ||
| public async shutdown () { | ||
| await this.pgSubscriber.close() | ||
| } | ||
| } |
Oops, something went wrong.