diff --git a/subquery/docker-compose.yml b/subquery/docker-compose.yml index 35678669..83c7d67a 100644 --- a/subquery/docker-compose.yml +++ b/subquery/docker-compose.yml @@ -9,6 +9,8 @@ services: - 5432:5432 volumes: - .data/postgres:/var/lib/postgresql/data + - ./docker/pg.conf:/usr/share/postgresql/postgresql.conf + command: ["postgres", "-c", "config_file=/usr/share/postgresql/postgresql.conf"] environment: POSTGRES_PASSWORD: postgres shm_size: '1gb' @@ -36,8 +38,8 @@ services: - ${SUB_COMMAND:-} # set SUB_COMMAND env variable to "test" to run tests - -f=/app - --db-schema=app - - --workers=4 - - --batch-size=30 + - --workers=2 + - --batch-size=20 - --unfinalized-blocks=false - --unsafe - --block-confirmations=0 @@ -67,9 +69,9 @@ services: DB_DATABASE: postgres DB_HOST: postgres DB_PORT: 5432 - # NODE_ENV: production + NODE_ENV: production command: - --name=app #- --log-level=silent - --playground - - --indexer=http://subquery-node:3000 + - --indexer=http://subquery-node:3000 \ No newline at end of file diff --git a/subquery/docker/pg.conf b/subquery/docker/pg.conf new file mode 100644 index 00000000..ae548814 --- /dev/null +++ b/subquery/docker/pg.conf @@ -0,0 +1,38 @@ +# Custom conf for 4GB RAM and 2-4 cores +shared_buffers = 1GB +work_mem = 12MB +maintenance_work_mem = 128MB + +effective_cache_size = 3GB +random_page_cost = 1.1 +seq_page_cost = 1.0 + +max_connections = 50 +max_worker_processes = 4 +max_parallel_workers_per_gather = 2 +max_parallel_workers = 4 + +autovacuum = on +autovacuum_naptime = 20s +autovacuum_vacuum_scale_factor = 0.2 +autovacuum_analyze_scale_factor = 0.1 +autovacuum_max_workers = 2 + +checkpoint_timeout = 10min +checkpoint_completion_target = 0.7 +max_wal_size = 2GB +min_wal_size = 512MB + +logging_collector = on +log_directory = 'pg_log' +log_filename = 'postgresql-%Y-%m-%d.log' +log_statement = 'none' +log_min_duration_statement = 1000 + +# open as it is in a container +listen_addresses = '*' +port = 5432 + +# adjust depeding on the resources available +parallel_setup_cost = 1000 +parallel_tuple_cost = 0.1 \ No newline at end of file diff --git a/subquery/mainnet-click.ts b/subquery/mainnet-click.ts index 2ad465ce..da2d0c27 100644 --- a/subquery/mainnet-click.ts +++ b/subquery/mainnet-click.ts @@ -77,7 +77,7 @@ const project: EthereumProject = { kind: EthereumHandlerKind.Event, handler: "handleTransfer", filter: { - topics: ["Transfer(address from,address to,uint256 tokenId)"], + topics: ["Transfer(address,address,uint256)"], }, }, ], @@ -133,7 +133,7 @@ const project: EthereumProject = { kind: EthereumHandlerKind.Event, handler: "handleTransfer", filter: { - topics: ["Transfer(address from,address to,uint256 tokenId)"], + topics: ["Transfer(address,address,uint256)"], }, }, ], @@ -189,7 +189,7 @@ const project: EthereumProject = { kind: EthereumHandlerKind.Event, handler: "handleTransfer", filter: { - topics: ["Transfer(address from,address to,uint256 tokenId)"], + topics: ["Transfer(address,address,uint256)"], }, }, ], @@ -245,7 +245,7 @@ const project: EthereumProject = { kind: EthereumHandlerKind.Event, handler: "handleTransfer", filter: { - topics: ["Transfer(address from,address to,uint256 tokenId)"], + topics: ["Transfer(address,address,uint256)"], }, }, ], diff --git a/subquery/mainnet-complete.ts b/subquery/mainnet-complete.ts index 659fd005..f73c0090 100644 --- a/subquery/mainnet-complete.ts +++ b/subquery/mainnet-complete.ts @@ -144,7 +144,7 @@ const project: EthereumProject = { startBlock: 42332281, // This is the block that the contract was deployed on options: { abi: "Grants", - address: "0xdAdF329E8b30D878b139074De163D3A591aAB394", + address: "0x5855c486d2381ba41762876f18684951d5902829", }, assets: new Map([ [ @@ -197,7 +197,7 @@ const project: EthereumProject = { startBlock: 42332281, // This is the block that the contract was deployed on options: { abi: "GrantsMigration", - address: "0xF81b3b954221BeDcf762cd18FEc1A22D25016B2E", + address: "0x1A89C10456A78a41B55c3aEAcfc865E079bE5690", }, assets: new Map([ [ diff --git a/subquery/schema.graphql b/subquery/schema.graphql index 2019ea10..614f2c35 100644 --- a/subquery/schema.graphql +++ b/subquery/schema.graphql @@ -65,9 +65,9 @@ type ERC721Token @entity @compositeIndexes(fields: [["owner", "contract"]]) { transfers: [ERC721Transfer!]! @derivedFrom(field: "token") application: String duration: Int - captureDate: BigInt @index - longitude: Float @index - latitude: Float @index + captureDate: BigInt + longitude: Float + latitude: Float locationPrecision: String contentHash: String } @@ -239,6 +239,7 @@ type VestingSchedule @entity { period: BigInt periodCount: Int perPeriodAmount: BigInt + total: BigInt transaction: Transaction! cancelled: Boolean cancelTransaction: Transaction @@ -251,6 +252,8 @@ type VestingSchedule @entity { type VestingScheduleCanceled @entity { id: ID! beneficiary: Account! @index + start: BigInt + end: BigInt cancelAuthority: Account @index affectedVestingSchedules: [String!] transaction: Transaction! @@ -259,6 +262,8 @@ type VestingScheduleCanceled @entity { type VestingScheduleRenounced @entity { id: ID! beneficiary: Account! @index + start: BigInt + end: BigInt cancelAuthority: Account @index affectedVestingSchedules: [String!] transaction: Transaction! @@ -267,6 +272,8 @@ type VestingScheduleRenounced @entity { type VestingScheduleClaimed @entity { id: ID! beneficiary: Account! @index + start: BigInt + end: BigInt amount: BigInt transaction: Transaction! } @@ -284,9 +291,9 @@ type Reward @entity { amount: BigInt @index transaction: Transaction! BatchMetadata: BatchMetadata - sequence: BigInt @index - status: Boolean @index - signature: String @index + sequence: BigInt + status: Boolean + signature: String } type AccountSnapshot @entity { @@ -328,10 +335,10 @@ type TokenSnapshotV2 @entity { tokensMinted: Int! fistBlockTimestamp: BigInt! @index latestBlockTimestamp: BigInt @index - latestBlockNumber: Int @index - firstBlockNumber: Int! @index - firstToken: ERC721Token! @index - latestToken: ERC721Token @index + latestBlockNumber: Int + firstBlockNumber: Int! + firstToken: ERC721Token! + latestToken: ERC721Token } type ERC20TransferV2 implements Event diff --git a/subquery/src/mappings/click-content-sign.ts b/subquery/src/mappings/click-content-sign.ts index d82822ca..adcde705 100644 --- a/subquery/src/mappings/click-content-sign.ts +++ b/subquery/src/mappings/click-content-sign.ts @@ -11,7 +11,7 @@ import { SafeMintTransaction, } from "../types/abi-interfaces/ClickContentSignAbi"; import { fetchAccount, fetchMetadata, fetchTransaction } from "../utils/utils"; -import { contractForSnapshot } from "../utils/const"; +import { contractForSnapshot, nodleContracts } from "../utils/const"; import { TokenSnapshot, TokenSnapshotV2 } from "../types"; const keysMapping = { @@ -85,8 +85,9 @@ export async function handleApprovalForAll(event: ApprovalForAllLog) { } export async function handleSafeMint(tx: SafeMintTransaction) { - assert(tx.args, "No tx.args"); - assert(tx.logs, "No tx.logs"); + if (!tx.args || !tx.logs) { + throw new Error("No tx.args or tx.logs"); + } // Call to the contract const contract = await fetchContract(String(tx.to).toLowerCase()); @@ -123,12 +124,10 @@ export async function handleSafeMint(tx: SafeMintTransaction) { token.uri = uri; - if (false) { + if (nodleContracts.includes(contract.id)) { const metadata = await fetchMetadata(uri, [ - "nodle-community-nfts.myfilebase.com", - "pinning.infura-ipfs.io", - "nodle-web-wallet.infura-ipfs.io", - "cloudflare-ipfs.com", + "nodle-community-nfts.myfilebase.com/ipfs", + "storage.googleapis.com/ipfs-backups", ]); if (metadata) { diff --git a/subquery/src/mappings/grants.ts b/subquery/src/mappings/grants.ts index af73f24f..9a749430 100644 --- a/subquery/src/mappings/grants.ts +++ b/subquery/src/mappings/grants.ts @@ -21,6 +21,8 @@ export async function handleClaimed(event: ClaimedLog): Promise { const who = event.args.who?.toString(); const amount = event.args.amount?.toBigInt(); const id = event.transactionHash + "-" + event.logIndex.toString(); + const start = event.args.start?.toBigInt(); + const end = event.args.end?.toBigInt(); const transaction = await fetchTransaction( event.transactionHash, @@ -31,6 +33,8 @@ export async function handleClaimed(event: ClaimedLog): Promise { const claimTransaction = new VestingScheduleClaimed(id, who, transaction.id); claimTransaction.amount = amount; + claimTransaction.start = start; + claimTransaction.end = end; await claimTransaction.save(); } @@ -43,6 +47,8 @@ export async function handleRenounced(event: RenouncedLog): Promise { const to = event.args.to.toString(); const from = event.args.from.toString(); + const start = event.args.start?.toBigInt(); + const end = event.args.end?.toBigInt(); const id = event.transactionHash + "-" + event.logIndex.toString(); const transaction = await fetchTransaction( event.transactionHash, @@ -52,6 +58,9 @@ export async function handleRenounced(event: RenouncedLog): Promise { const action = new VestingScheduleRenounced(id, to, transaction.id); + action.start = start; + action.end = end; + const schedules = await VestingSchedule.getByCancelAuthorityId(from, { limit: 100 }); if (schedules && schedules.length > 0) { @@ -82,7 +91,6 @@ export async function handleVestingScheduleAdded( const id = event.transactionHash + "-" + event.logIndex.toString(); const to = event.args.to.toString(); const schedule = event.args.schedule; - logger.info("schedule: " + JSON.stringify(schedule) + " to: " + to); const transaction = await fetchTransaction( event.transactionHash, event.block.timestamp, @@ -90,11 +98,13 @@ export async function handleVestingScheduleAdded( ); const vestingSchedule = new VestingSchedule(id, to, transaction.id); - + const total = + BigInt(schedule.periodCount) * schedule.perPeriodAmount.toBigInt(); vestingSchedule.start = schedule.start.toBigInt(); vestingSchedule.period = schedule.period.toBigInt(); vestingSchedule.periodCount = schedule.periodCount; vestingSchedule.perPeriodAmount = schedule.perPeriodAmount.toBigInt(); + vestingSchedule.total = total; vestingSchedule.cancelAuthorityId = schedule.cancelAuthority?.toString(); @@ -111,6 +121,8 @@ export async function handleVestingSchedulesCanceled( const from = event.args.from.toString(); const to = event.args.to.toString(); + const start = event.args.start?.toBigInt(); + const end = event.args.end?.toBigInt(); const id = event.transactionHash + "-" + event.logIndex.toString(); const transaction = await fetchTransaction( @@ -121,6 +133,9 @@ export async function handleVestingSchedulesCanceled( const action = new VestingScheduleCanceled(id, to, transaction.id); + action.start = start; + action.end = end; + const schedules = await VestingSchedule.getByCancelAuthorityId(from, { limit: 100 }); if (schedules && schedules.length > 0) { diff --git a/subquery/src/mappings/migration-nft.ts b/subquery/src/mappings/migration-nft.ts index 3d4fa7f1..16fd58e7 100644 --- a/subquery/src/mappings/migration-nft.ts +++ b/subquery/src/mappings/migration-nft.ts @@ -35,7 +35,6 @@ export async function handleNFTTransfer(event: TransferLog): Promise { ]).catch((error) => { return null; }); - logger.info("Token URI: " + String(tokenUri)); if (tokenUri && nodleContracts.includes(contract.id)) { const metadata = await fetchMetadata(String(tokenUri), [ @@ -95,7 +94,6 @@ export async function handleApproval(event: ApprovalLog): Promise { ]).catch((error) => { return null; }); - logger.info("Token URI: " + String(tokenUri)); if (tokenUri && nodleContracts.includes(contract.id)) { const metadata = await fetchMetadata(String(tokenUri), [ "nodle-community-nfts.myfilebase.com/ipfs", diff --git a/subquery/src/utils/erc20.ts b/subquery/src/utils/erc20.ts index 3caefc9c..2412df4d 100644 --- a/subquery/src/utils/erc20.ts +++ b/subquery/src/utils/erc20.ts @@ -9,9 +9,6 @@ export const fetchContract = async ( const contract = await ERC20Contract.get(lowercaseAddress); if (!contract) { - logger.error( - `Contract not found for lowercaseAddress: ${lowercaseAddress}` - ); const newContract = new ERC20Contract(lowercaseAddress, lowercaseAddress); newContract.save(); diff --git a/subquery/src/utils/erc721.ts b/subquery/src/utils/erc721.ts index 18a6383f..6dc3afbc 100644 --- a/subquery/src/utils/erc721.ts +++ b/subquery/src/utils/erc721.ts @@ -11,9 +11,6 @@ export const fetchContract = async ( const contract = await ERC721Contract.get(lowercaseAddress); if (!contract) { - logger.error( - `Contract not found for lowercaseAddress: ${lowercaseAddress}` - ); const newContract = new ERC721Contract(lowercaseAddress, lowercaseAddress); newContract.save(); @@ -33,7 +30,6 @@ export const fetchToken = async ( const token = await ERC721Token.get(id); if (!token) { - logger.error(`Token not found for id: ${id}`); const newToken = new ERC721Token( id, contractId, @@ -54,7 +50,9 @@ export const getApprovalLog = ( address: string ) => { const targetLog = logs.find((log) => log.args?.to === address); - assert(targetLog, "No target log found"); + if (!targetLog) { + throw new Error("Approval log not found"); + } return targetLog.args; }; @@ -73,7 +71,6 @@ export const fetchERC721Operator = async ( const op = await ERC721Operator.get(id); if (!op) { - logger.error(`Operator not found for id: ${id}`); const newOp = new ERC721Operator( id, contract.id, diff --git a/subquery/src/utils/migration.ts b/subquery/src/utils/migration.ts index 63660cff..888feeba 100644 --- a/subquery/src/utils/migration.ts +++ b/subquery/src/utils/migration.ts @@ -9,9 +9,6 @@ export const fetchContract = async ( const contract = await ProposalContract.get(lowercaseAddress); if (!contract) { - logger.error( - `Contract not found for lowercaseAddress: ${lowercaseAddress}` - ); const newContract = new ProposalContract( lowercaseAddress, lowercaseAddress diff --git a/subquery/src/utils/utils.ts b/subquery/src/utils/utils.ts index aac223b1..11cb4159 100644 --- a/subquery/src/utils/utils.ts +++ b/subquery/src/utils/utils.ts @@ -5,10 +5,11 @@ export async function fetchAccount( address: string, timestamp?: bigint ): Promise { - let account = await Account.get(String(address).toLowerCase()); + const lowercaseAddress = String(address).toLowerCase(); + let account = await Account.get(lowercaseAddress); if (!account) { - account = new Account(address); + account = new Account(lowercaseAddress); account.timestamp = timestamp || BigInt(0); account.balance = BigInt(0); account.save(); @@ -25,7 +26,6 @@ export const fetchTransaction = async ( const tx = await Transaction.get(txHash); if (!tx) { - logger.error(`Transaction not found for hash: ${txHash}`); const newTx = new Transaction(txHash, timestamp, blocknumber); newTx.save(); diff --git a/subquery/test.ts b/subquery/test.ts index 08aafdeb..31c3977c 100644 --- a/subquery/test.ts +++ b/subquery/test.ts @@ -114,8 +114,6 @@ export const getContractDetails = async ( const [isErc721] = await callContract(address, abi, "supportsInterface", [ "0x80ac58cd", ]).catch((error: any) => { - logger.info(`Error calling supportsInterface for ${address}`); - logger.info(JSON.stringify(error)); return [false]; }); @@ -124,14 +122,9 @@ export const getContractDetails = async ( const [erc1155] = await callContract(address, abi, "supportsInterface", [ "0xd9b67a26", ]).catch((error: any) => { - logger.info(`Error calling supportsInterface for ${address}`); - logger.info(JSON.stringify(error)); return [false]; }); - console.log("erc1155", erc1155); - - logger.info(`isErc721: ${isErc721}`); const isErc20 = isErc721 || erc1155 ? false : await checkERC20(address); return { @@ -141,8 +134,6 @@ export const getContractDetails = async ( isErc20: Boolean(isErc20), }; } catch (error: any) { - logger.info(`Error getting contract details for ${address}`); - logger.info(JSON.stringify(error)); return { symbol: "", diff --git a/subquery/testnet-complete.ts b/subquery/testnet-complete.ts index a7e5259c..6ff71fef 100644 --- a/subquery/testnet-complete.ts +++ b/subquery/testnet-complete.ts @@ -28,7 +28,10 @@ const project: EthereumProject = { }, network: { chainId: "300", // zKsync sepolia testnet - endpoint: [process.env.ZKSYNC_TESTNET_RPC!], + endpoint: [ + process.env.ZKSYNC_TESTNET_RPC!, + "https://sepolia.era.zksync.dev", + ], }, dataSources: [ { @@ -179,7 +182,7 @@ const project: EthereumProject = { kind: EthereumHandlerKind.Event, handler: "handleNFTTransfer", filter: { - topics: ["Transfer (address from, address to, uint256 tokenId)"], + topics: ["Transfer(address,address,uint256)"], }, }, ], @@ -191,7 +194,7 @@ const project: EthereumProject = { startBlock: 3548139, // This is the block that the contract was deployed on options: { abi: "Grants", - address: "0xdAdF329E8b30D878b139074De163D3A591aAB394", + address: "0x66f762DB62E5D8609317436e8F2784c5ACBC9c61", }, assets: new Map([ [ @@ -244,7 +247,7 @@ const project: EthereumProject = { startBlock: 3548120, // This is the block that the contract was deployed on options: { abi: "GrantsMigration", - address: "0xF81b3b954221BeDcf762cd18FEc1A22D25016B2E", + address: "0xED90FDAB958AC7e4942f51b9175B76c8e181c5Cb", }, assets: new Map([ [