Cosmos codegen, protobuf to ts (subquery#1958)

* wip, draft cosmos codegen via protobuf

* add tests for existing cosmos codegen, update linting to ignore .proto

* add more tests for pathing, update generate structure, clear logs

* add validate on cosmos manfiest

* update logic and add commonProtos

* migrate dependencies to common-cosmos

* update changelog

* update with dev common-cosmos, clean functions

* remove test protos, fix tests

* improve manifest typing

* remove yalc orphan and array defination

* removed unused array interface

* bump common cosmos version

* bump common-cosmos to 2.4.1

* rebase main

.eslintignore

@@ -6,3 +6,4 @@
 packages/**/dist/**
 packages/**/lib/**
 .eslintrc.js
+*.proto

.prettierignore

@@ -10,3 +10,4 @@
 *.cmd
 *.sh
 generate-project*.yaml
+*.proto

packages/cli/CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- `codegen` support for Cosmos (Protobufs to ts) (#1958)
+
 ### Fixed
 - Publish command uploading files to IPFS multiple times (#1967)
 

packages/cli/src/controller/codegen-controller.ts

@@ -9,6 +9,10 @@ import {
   isRuntimeCosmosDs,
   RuntimeDatasourceTemplate as CosmosDsTemplate,
   CustomDatasourceTemplate as CosmosCustomDsTemplate,
+  generateProto,
+  tempProtoDir,
+  validateCosmosManifest,
+  ProjectManifestImpls as CosmosManifest,
 } from '@subql/common-cosmos';
 import {
   isCustomDs as isCustomEthereumDs,
@@ -425,6 +429,14 @@ export async function codegen(projectPath: string, fileNames: string[] = [DEFAUL
     datasources = datasources.concat(customDatasources);
   }
 
+  const chainTypes = plainManifests
+    .filter((m) => validateCosmosManifest(m))
+    .map((m) => (m as CosmosManifest).network.chainTypes);
+
+  if (chainTypes.length !== 0) {
+    await generateProto(chainTypes, projectPath, prepareDirPath, renderTemplate, upperFirst, tempProtoDir);
+  }
+
   await generateAbis(datasources, projectPath);
 
   if (exportTypes.interfaces || exportTypes.models || exportTypes.enums || exportTypes.datasources) {

packages/cli/src/controller/codegen-cosmos.test.ts

@@ -0,0 +1,90 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import fs from 'fs';
+import path from 'path';
+import {promisify} from 'util';
+import {generateProto, tempProtoDir} from '@subql/common-cosmos';
+import {upperFirst} from 'lodash';
+import rimraf from 'rimraf';
+import {prepareDirPath, renderTemplate} from '../utils';
+
+const PROJECT_PATH = path.join(__dirname, '../../test/protoTest1');
+const MOCK_CHAINTYPES = [
+  {
+    'osmosis.gamm.v1beta1': {
+      file: './proto/osmosis/gamm/v1beta1/tx.proto',
+      messages: ['MsgSwapExactAmountIn'],
+    },
+  },
+  {
+    'osmosis.poolmanager.v1beta1': {
+      file: './proto/osmosis/poolmanager/v1beta1/swap_route.proto',
+      messages: ['SwapAmountInRoute'],
+    },
+  },
+];
+
+describe('Able to generate cosmos types from protobuf', () => {
+  afterEach(async () => {
+    await promisify(rimraf)(path.join(__dirname, '../../test/protoTest1/src'));
+    await promisify(rimraf)(path.join(__dirname, '../../test/protoTest2/src'));
+  });
+
+  it('Able to generate ts types from protobufs', async () => {
+    const expectedGeneratedCode =
+      '' +
+      `// SPDX-License-Identifier: Apache-2.0
+
+// Auto-generated , DO NOT EDIT
+import {CosmosMessage} from "@subql/types-cosmos";
+
+import {MsgSwapExactAmountIn} from "./proto-interfaces/osmosis/gamm/v1beta1/tx";
+
+import {SwapAmountInRoute} from "./proto-interfaces/osmosis/poolmanager/v1beta1/swap_route";
+
+
+export type MsgSwapExactAmountInMessage = CosmosMessage<MsgSwapExactAmountIn>;
+
+export type SwapAmountInRouteMessage = CosmosMessage<SwapAmountInRoute>;
+`;
+    await generateProto(MOCK_CHAINTYPES, PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst, tempProtoDir);
+    const codegenResult = await fs.promises.readFile(path.join(PROJECT_PATH, '/src/types/CosmosMessageTypes.ts'));
+    expect(fs.existsSync(`${PROJECT_PATH}/src/types/CosmosMessageTypes.ts`)).toBeTruthy();
+    expect(codegenResult.toString()).toBe(expectedGeneratedCode);
+  });
+  it('On missing protobuf dependency should throw', async () => {
+    const tmpDir = await tempProtoDir(PROJECT_PATH);
+    const badChainTypes = [
+      {
+        'osmosis.gamm.v1beta1': {
+          file: './proto/cosmos/osmosis/gamm/v1beta1/tx.proto',
+          messages: ['MsgSwapExactAmountIn'],
+        },
+      },
+    ];
+    await expect(
+      generateProto(badChainTypes, PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst, () =>
+        Promise.resolve(tmpDir)
+      )
+    ).rejects.toThrow(
+      'Failed to generate from protobufs. Error: chainType osmosis.gamm.v1beta1, file ./proto/cosmos/osmosis/gamm/v1beta1/tx.proto does not exist'
+    );
+    expect(fs.existsSync(tmpDir)).toBe(false);
+  });
+  it('create temp dir with all protobufs', async () => {
+    // user Protobufs should not be overwritten
+    const preFile = await fs.promises.readFile(path.join(PROJECT_PATH, 'proto/osmosis/gamm/v1beta1/tx.proto'));
+    const tmpDir = await tempProtoDir(PROJECT_PATH);
+    const afterFile = await fs.promises.readFile(path.join(tmpDir, 'osmosis/gamm/v1beta1/tx.proto'));
+    expect(preFile.toString()).toBe(afterFile.toString());
+    await promisify(rimraf)(tmpDir);
+  });
+  it('tmpDirectory should be removed after codegen', async () => {
+    const tmpDir = await tempProtoDir(PROJECT_PATH);
+    await generateProto(MOCK_CHAINTYPES, PROJECT_PATH, prepareDirPath, renderTemplate, upperFirst, () =>
+      Promise.resolve(tmpDir)
+    );
+    expect(fs.existsSync(tmpDir)).toBe(false);
+  });
+});

packages/cli/test/protoTest1/project.yaml

@@ -0,0 +1,45 @@
+specVersion: 1.0.0
+name: test
+version: 0.0.1
+runner:
+  node:
+    name: '@subql/node-cosmos'
+    version: '*'
+  query:
+    name: '@subql/query'
+    version: '*'
+description: ''
+repository: ''
+schema:
+  file: ./schema.graphql
+network:
+  chainId: osmosis-1
+  endpoint:
+    - https://osmosis.api.onfinality.io/public
+  dictionary: 'https://api.subquery.network/sq/subquery/cosmos-osmosis-dictionary'
+  chainTypes:
+    osmosis.gamm.v1beta1:
+      file: './proto/osmosis/gamm/v1beta1/tx.proto'
+      messages:
+        - MsgSwapExactAmountIn
+    osmosis.poolmanager.v1beta1:
+      # needed by MsgSwapExactAmountIn
+      file: './proto/osmosis/poolmanager/v1beta1/swap_route.proto'
+      messages:
+        - SwapAmountInRoute
+    cosmos.base.v1beta1:
+      # needed by MsgSwapExactAmountIn
+      file: './proto/cosmos/base/v1beta1/coin.proto'
+      messages:
+        - 'Coin'
+
+dataSources:
+  - kind: cosmos/Runtime
+    startBlock: 9798050
+    mapping:
+      file: ./dist/index.js
+      handlers:
+        - handler: handleMessage
+          kind: cosmos/MessageHandler
+          filter:
+            type: /osmosis.gamm.v1beta1.MsgSwapExactAmountIn

packages/cli/test/protoTest1/proto/cosmos/base/v1beta1/coin.proto

@@ -0,0 +1,43 @@
+syntax = "proto3";
+package cosmos.base.v1beta1;
+
+import "gogoproto/gogo.proto";
+import "cosmos_proto/cosmos.proto";
+
+option go_package                       = "github.com/cosmos/cosmos-sdk/types";
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all)         = false;
+
+// Coin defines a token with a denomination and an amount.
+//
+// NOTE: The amount field is an Int which implements the custom method
+// signatures required by gogoproto.
+message Coin {
+  option (gogoproto.equal) = true;
+
+  string denom  = 1;
+  string amount = 2
+      [(cosmos_proto.scalar) = "cosmos.Int", (gogoproto.customtype) = "Int", (gogoproto.nullable) = false];
+}
+
+// DecCoin defines a token with a denomination and a decimal amount.
+//
+// NOTE: The amount field is an Dec which implements the custom method
+// signatures required by gogoproto.
+message DecCoin {
+  option (gogoproto.equal) = true;
+
+  string denom  = 1;
+  string amount = 2
+      [(cosmos_proto.scalar) = "cosmos.Dec", (gogoproto.customtype) = "Dec", (gogoproto.nullable) = false];
+}
+
+// IntProto defines a Protobuf wrapper around an Int object.
+message IntProto {
+  string int = 1 [(cosmos_proto.scalar) = "cosmos.Int", (gogoproto.customtype) = "Int", (gogoproto.nullable) = false];
+}
+
+// DecProto defines a Protobuf wrapper around a Dec object.
+message DecProto {
+  string dec = 1 [(cosmos_proto.scalar) = "cosmos.Dec", (gogoproto.customtype) = "Dec", (gogoproto.nullable) = false];
+}