Move code to node core, tidy up some unused deps (subquery#2357)

* Move code to node core, tidy up some unused deps * Attempt to fix migration service test * Fix connection pool tests * Change postgres test host * Revert postgress change add logging to test * Remove paralell from build option * Enable btree gist in CI * Move ds processor code to node-core/types-core * Remove commented code * Update changelogs * Remove unused imports * Make sandbox service more extensible * Simplify and make test more reliable * Fix exports from tests
dapplooker · Apr 17, 2024 · 17d0e0b · 17d0e0b
1 parent 7e9a725
commit 17d0e0b
Show file tree

Hide file tree

Showing 43 changed files with 477 additions and 417 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -49,6 +49,9 @@ jobs:
     steps:
         - uses: actions/checkout@v4
 
+        - name: Enable btree btree_gist
+          run: psql "postgresql://$DB_USER:$DB_PASS@$DB_HOST:$DB_PORT/$DB_DATABASE" -c "CREATE EXTENSION IF NOT EXISTS btree_gist;"
+
         - name: Use Node 18
           uses: actions/setup-node@v4
           with:

diff --git a/jest.config.js b/jest.config.js
@@ -168,9 +168,10 @@ module.exports = {
   // ],
 
   // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
-  // testPathIgnorePatterns: [
-  //   "/node_modules/"
-  // ],
+  testPathIgnorePatterns: [
+    '.*\\.fixtures\\.ts$',
+    "/node_modules/"
+  ],
 
   // The regexp pattern or array of patterns that Jest uses to detect test files
   testRegex: '.*\\.spec\\.ts$',

diff --git a/package.json b/package.json
@@ -41,7 +41,7 @@
     "node-fetch": "2.6.7"
   },
   "scripts": {
-    "build": "yarn workspaces foreach -ptA run build",
+    "build": "yarn workspaces foreach -tA run build",
     "lint": "eslint packages --ext .ts",
     "test": "jest --coverage",
     "test:ci": "jest --testRegex='.*\\.(spec|test)\\.ts$'",

diff --git a/packages/cli/src/commands/publish.test.ts b/packages/cli/src/commands/publish.test.ts
@@ -5,12 +5,18 @@ import fs from 'fs';
 import path from 'path';
 import {promisify} from 'util';
 import rimraf from 'rimraf';
-import {createTestProject, projectSpecV1_0_0} from '../controller/publish-controller.spec';
+import {createTestProject} from '../createProject.fixtures';
 import Publish from './publish';
 
+jest.setTimeout(300_000); // 300s
 describe('Intergration test - Publish', () => {
   let projectDir: string;
-  afterEach(async () => {
+
+  beforeAll(async () => {
+    projectDir = await createTestProject();
+  });
+
+  afterAll(async () => {
     try {
       if (!projectDir) return;
       await promisify(rimraf)(projectDir);
@@ -21,16 +27,24 @@ describe('Intergration test - Publish', () => {
 
   it('overwrites any exisiting CID files', async () => {
     const initCID = 'QmWLxg7xV7ZWUyc7ZxZ8XuQQ7NmH8WQGXzg7VZ3QQNqF-testing';
-    projectDir = await createTestProject(projectSpecV1_0_0);
     const cidFilePath = path.resolve(projectDir, '.project-cid');
     await fs.promises.writeFile(cidFilePath, initCID);
     await Publish.run(['-f', projectDir, '-o']);
     const cidValue = await fs.promises.readFile(cidFilePath, 'utf8');
     expect(initCID).not.toEqual(cidValue);
   });
 
+  it('create ipfsCID file stored in local with dictiory path', async () => {
+    await Publish.run(['-f', projectDir]);
+    const cidFile = path.resolve(projectDir, '.project-cid');
+    const fileExists = fs.existsSync(cidFile);
+    const IPFScontent = await fs.promises.readFile(cidFile, 'utf8');
+    expect(IPFScontent).toBeDefined();
+    expect(fileExists).toBeTruthy();
+  });
+
+  // Run this last because it modifies the project
   it('file name consistent with manfiest file name, if -f <manifest path> is used', async () => {
-    projectDir = await createTestProject(projectSpecV1_0_0);
     const manifestPath = path.resolve(projectDir, 'project.yaml');
     const testManifestPath = path.resolve(projectDir, 'test.yaml');
     fs.renameSync(manifestPath, testManifestPath);
@@ -40,14 +54,4 @@ describe('Intergration test - Publish', () => {
     const fileExists = await fs.promises.stat(cidFile);
     expect(fileExists.isFile()).toBeTruthy();
   });
-
-  it('create ipfsCID file stored in local with dictiory path', async () => {
-    projectDir = await createTestProject(projectSpecV1_0_0);
-    await Publish.run(['-f', projectDir]);
-    const cidFile = path.resolve(projectDir, '.project-cid');
-    const fileExists = fs.existsSync(cidFile);
-    const IPFScontent = await fs.promises.readFile(cidFile, 'utf8');
-    expect(IPFScontent).toBeDefined();
-    expect(fileExists).toBeTruthy();
-  });
 });
diff --git a/packages/cli/src/controller/publish-controller.spec.ts b/packages/cli/src/controller/publish-controller.spec.ts
@@ -1,83 +1,28 @@
 // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
-import childProcess from 'child_process';
 import fs from 'fs';
-import os from 'os';
 import path from 'path';
 import {promisify} from 'util';
 import {DEFAULT_MANIFEST, mapToObject, ReaderFactory, toJsonObject} from '@subql/common';
-import {parseSubstrateProjectManifest, ProjectManifestV1_0_0Impl} from '@subql/common-substrate';
-import {create} from 'ipfs-http-client';
+import {parseSubstrateProjectManifest} from '@subql/common-substrate';
 import rimraf from 'rimraf';
-import Build from '../commands/build';
-import Codegen from '../commands/codegen';
 import Publish from '../commands/publish';
-import {ProjectSpecBase, ProjectSpecV0_0_1, ProjectSpecV0_2_0, ProjectSpecV1_0_0} from '../types';
-import {cloneProjectTemplate, fetchExampleProjects, prepare} from './init-controller';
-import {uploadFile, uploadToIpfs} from './publish-controller';
+import {createTestProject} from '../createProject.fixtures';
+import {uploadToIpfs} from './publish-controller';
 
-const projectSpecV0_0_1: ProjectSpecV0_0_1 = {
-  name: 'mocked_starter',
-  endpoint: 'wss://rpc.polkadot.io/public-ws',
-  author: 'jay',
-  description: 'this is test for init controller',
-};
-
-const projectSpecV0_2_0: ProjectSpecV0_2_0 = {
-  name: 'mocked_starter',
-  genesisHash: '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3',
-  endpoint: 'wss://rpc.polkadot.io/public-ws',
-  author: 'jay',
-  description: 'this is test for init controller',
-};
-
-// eslint-disable-next-line jest/no-export
-export const projectSpecV1_0_0: ProjectSpecV1_0_0 = {
-  name: 'mocked_starter',
-  chainId: '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3',
-  endpoint: 'wss://rpc.polkadot.io/public-ws',
-  author: 'jay',
-  description: 'this is test for init controller',
-  runner: {
-    node: {
-      name: '@subql/node',
-      version: '>=1.0.0',
-    },
-    query: {
-      name: '@subql/query',
-      version: '*',
-    },
-  },
-};
-
-const ipfsEndpoint = 'http://localhost:5001/api/v0';
 // Replace/Update your access token when test locally
 const testAuth = process.env.SUBQL_ACCESS_TOKEN;
 
-jest.setTimeout(150000);
-
-// eslint-disable-next-line jest/no-export
-export async function createTestProject(projectSpec: ProjectSpecBase): Promise<string> {
-  const tmpdir = await fs.promises.mkdtemp(`${os.tmpdir()}${path.sep}`);
-  const projectDir = path.join(tmpdir, projectSpec.name);
-  const projects = await fetchExampleProjects('polkadot', 'polkadot');
-  const projectPath = await cloneProjectTemplate(tmpdir, projectSpec.name, projects[0]);
-  await prepare(projectPath, projectSpec);
-
-  // Install dependencies
-  childProcess.execSync(`npm i`, {cwd: projectDir});
-
-  await Codegen.run(['-l', projectDir]);
-  await Build.run(['-f', projectDir]);
-
-  return projectDir;
-}
-
+jest.setTimeout(300_000); // 300s
 describe('Cli publish', () => {
   let projectDir: string;
 
-  afterEach(async () => {
+  beforeAll(async () => {
+    projectDir = await createTestProject();
+  });
+
+  afterAll(async () => {
     try {
       if (!projectDir) return;
       await promisify(rimraf)(projectDir);
@@ -86,45 +31,12 @@ describe('Cli publish', () => {
     }
   });
 
-  it('should not allow uploading a v0.0.1 spec version project', async () => {
-    projectDir = await createTestProject(projectSpecV0_0_1);
-    await expect(uploadToIpfs([''], ipfsEndpoint, projectDir)).rejects.toBeDefined();
-  });
-
-  it(`upload file to ipfs`, async () => {
-    // only enable when test locally
-    const ipfs = create({url: ipfsEndpoint});
-
-    //test string
-    const cid = await uploadFile({path: '', content: 'Test for upload string to ipfs'}, testAuth);
-    console.log(`upload file cid: ${cid}`);
-  });
-
   it('should upload appropriate project to IPFS', async () => {
-    projectDir = await createTestProject(projectSpecV0_2_0);
     const cid = await uploadToIpfs([projectDir], testAuth);
     expect(cid).toBeDefined();
-    // validation no longer required, as it is deployment object been published
-    // await expect(Validate.run(['-l', cid, '--ipfs', ipfsEndpoint])).resolves.toBe(undefined);
-  });
-
-  it('upload project from a manifest', async () => {
-    projectDir = await createTestProject(projectSpecV0_2_0);
-    const manifestPath = path.resolve(projectDir, DEFAULT_MANIFEST);
-    const testManifestPath = path.resolve(projectDir, 'test.yaml');
-    fs.renameSync(manifestPath, testManifestPath);
-    await expect(Publish.run(['-f', testManifestPath])).resolves.not.toThrow();
-  });
-
-  it('v1.0.0 should deploy', async () => {
-    projectDir = await createTestProject(projectSpecV1_0_0);
-    const reader = await ReaderFactory.create(projectDir);
-    const manifest = parseSubstrateProjectManifest(await reader.getProjectSchema()).asImpl;
-    expect((manifest as ProjectManifestV1_0_0Impl).runner).toBeDefined();
   });
 
   it('convert to deployment and removed descriptive field', async () => {
-    projectDir = await createTestProject(projectSpecV1_0_0);
     const reader = await ReaderFactory.create(projectDir);
     const manifest = parseSubstrateProjectManifest(await reader.getProjectSchema());
     const deployment = manifest.toDeployment();

diff --git a/packages/cli/src/createProject.fixtures.ts b/packages/cli/src/createProject.fixtures.ts
@@ -0,0 +1,61 @@
+// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import childProcess from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import Build from './commands/build';
+import Codegen from './commands/codegen';
+import {cloneProjectTemplate, ExampleProjectInterface, prepare} from './controller/init-controller';
+import {ProjectSpecV1_0_0} from './types';
+
+const projectSpecV1_0_0: ProjectSpecV1_0_0 = {
+  name: 'mocked_starter',
+  chainId: '0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3',
+  endpoint: 'wss://rpc.polkadot.io/public-ws',
+  author: 'jay',
+  description: 'this is test for init controller',
+  runner: {
+    node: {
+      name: '@subql/node',
+      version: '>=1.0.0',
+    },
+    query: {
+      name: '@subql/query',
+      version: '*',
+    },
+  },
+};
+
+async function getExampleProject(networkFamily: string, network: string): Promise<ExampleProjectInterface | undefined> {
+  const res = await fetch('https://raw.githubusercontent.com/subquery/templates/main/dist/output.json');
+
+  if (!res.ok) {
+    throw new Error('Failed to get template');
+  }
+
+  const templates = (await res.json()) as {
+    code: string;
+    networks: {code: string; examples: ExampleProjectInterface[]}[];
+  }[];
+  return templates.find((t) => t.code === networkFamily)?.networks.find((n) => n.code === network)?.examples[0];
+}
+
+export async function createTestProject(): Promise<string> {
+  const tmpdir = await fs.promises.mkdtemp(`${os.tmpdir()}${path.sep}`);
+  const projectDir = path.join(tmpdir, projectSpecV1_0_0.name);
+
+  const exampleProject = await getExampleProject('polkadot', 'polkadot');
+
+  const projectPath = await cloneProjectTemplate(tmpdir, projectSpecV1_0_0.name, exampleProject);
+  await prepare(projectPath, projectSpecV1_0_0);
+
+  // Install dependencies
+  childProcess.execSync(`npm i`, {cwd: projectDir});
+
+  await Codegen.run(['-l', projectDir]);
+  await Build.run(['-f', projectDir]);
+
+  return projectDir;
+}
diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md
@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Various pieces of code from node and generalised them (#2357)
+
+### Changed
+- Tidy up block dispatcher constructor args (#2357)
 
 ## [9.0.0] - 2024-04-12
 ### Added

diff --git a/packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts b/packages/node-core/src/db/migration-service/SchemaMigration.service.test.ts
@@ -76,9 +76,6 @@ describe('SchemaMigration integration tests', () => {
 
   afterEach(async () => {
     await sequelize.dropSchema(schemaName, {logging: false});
-  });
-
-  afterAll(async () => {
     await sequelize?.close();
   });
 

diff --git a/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/base-block-dispatcher.ts
@@ -9,7 +9,6 @@ import {NodeConfig, IProjectUpgradeService} from '../../configure';
 import {IndexerEvent, PoiEvent} from '../../events';
 import {getLogger} from '../../logger';
 import {IQueue, mainThreadOnly} from '../../utils';
-import {DynamicDsService} from '../dynamic-ds.service';
 import {PoiBlock, PoiSyncService} from '../poi';
 import {SmartBatchService} from '../smartBatch.service';
 import {StoreService} from '../store.service';
@@ -51,19 +50,21 @@ export abstract class BaseBlockDispatcher<Q extends IQueue, DS, B> implements IB
   protected currentProcessingHeight = 0;
   private _onDynamicDsCreated?: (height: number) => Promise<void>;
 
+  protected smartBatchService: SmartBatchService;
+
   constructor(
     protected nodeConfig: NodeConfig,
     protected eventEmitter: EventEmitter2,
     private project: ISubqueryProject,
     protected projectService: IProjectService<DS>,
     private projectUpgradeService: IProjectUpgradeService,
     protected queue: Q,
-    protected smartBatchService: SmartBatchService,
     protected storeService: StoreService,
     private storeCacheService: StoreCacheService,
-    private poiSyncService: PoiSyncService,
-    protected dynamicDsService: DynamicDsService<any>
-  ) {}
+    private poiSyncService: PoiSyncService
+  ) {
+    this.smartBatchService = new SmartBatchService(nodeConfig.batchSize);
+  }
 
   abstract enqueueBlocks(heights: (IBlock<B> | number)[], latestBufferHeight?: number): void | Promise<void>;
 
@@ -230,6 +231,7 @@ export abstract class BaseBlockDispatcher<Q extends IQueue, DS, B> implements IB
     if (isNullMerkelRoot(operationHash)) {
       return;
     }
+
     const poiBlock = PoiBlock.create(height, blockHash, operationHash, this.project.id);
     // This is the first creation of POI
     this.poi.bulkUpsert([poiBlock]);

diff --git a/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts b/packages/node-core/src/indexer/blockDispatcher/block-dispatcher.ts
@@ -12,8 +12,6 @@ import {getBlockHeight, IBlock, PoiSyncService} from '../../indexer';
 import {getLogger} from '../../logger';
 import {profilerWrap} from '../../profiler';
 import {Queue, AutoQueue, delay, memoryLock, waitForBatchSize, isTaskFlushedError} from '../../utils';
-import {DynamicDsService} from '../dynamic-ds.service';
-import {SmartBatchService} from '../smartBatch.service';
 import {StoreService} from '../store.service';
 import {StoreCacheService} from '../storeCache';
 import {IProjectService, ISubqueryProject} from '../types';
@@ -45,12 +43,10 @@ export abstract class BlockDispatcher<B, DS>
     eventEmitter: EventEmitter2,
     projectService: IProjectService<DS>,
     projectUpgradeService: IProjectUpgradeService,
-    smartBatchService: SmartBatchService,
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiSyncService: PoiSyncService,
     project: ISubqueryProject,
-    dynamicDsService: DynamicDsService<DS>,
     fetchBlocksBatches: BatchBlockFetcher<B>
   ) {
     super(
@@ -60,11 +56,9 @@ export abstract class BlockDispatcher<B, DS>
       projectService,
       projectUpgradeService,
       new Queue(nodeConfig.batchSize * 3),
-      smartBatchService,
       storeService,
       storeCacheService,
-      poiSyncService,
-      dynamicDsService
+      poiSyncService
     );
     this.processQueue = new AutoQueue(nodeConfig.batchSize * 3, 1, nodeConfig.timeout, 'Process');
     this.fetchQueue = new AutoQueue(nodeConfig.batchSize * 3, nodeConfig.batchSize, nodeConfig.timeout, 'Fetch');