Merge branch 'master' into feat-add-subscription

package.json

@@ -32,7 +32,7 @@
     "@snapshot-labs/pineapple": "^1.1.0",
     "@snapshot-labs/snapshot-metrics": "^1.4.1",
     "@snapshot-labs/snapshot-sentry": "^1.5.5",
-    "@snapshot-labs/snapshot.js": "^0.12.32",
+    "@snapshot-labs/snapshot.js": "^0.12.41",
     "bluebird": "^3.7.2",
     "connection-string": "^1.0.1",
     "cors": "^2.8.5",

src/helpers/actions.ts

@@ -82,6 +82,7 @@ export async function sxSpaceExists(network: string, spaceId: string): Promise<b
     matic: 'https://api.studio.thegraph.com/query/23545/sx-polygon/version/latest',
     arb1: 'https://api.studio.thegraph.com/query/23545/sx-arbitrum/version/latest',
     oeth: 'https://api.studio.thegraph.com/query/23545/sx-optimism/version/latest',
+    base: 'https://api.studio.thegraph.com/query/23545/sx-base/version/latest',
     sn: 'https://api.snapshot.box',
     'sn-sep': 'https://testnet-api.snapshot.box',
     'linea-testnet':

src/helpers/utils.ts

@@ -8,7 +8,7 @@ import snapshot from '@snapshot-labs/snapshot.js';
 import { Response } from 'express';
 import fetch from 'node-fetch';
 
-const MAINNET_NETWORK_ID_WHITELIST = ['s', 'eth', 'matic', 'arb1', 'oeth', 'sn'];
+const MAINNET_NETWORK_ID_WHITELIST = ['s', 'eth', 'matic', 'arb1', 'oeth', 'sn', 'base'];
 const TESTNET_NETWORK_ID_WHITELIST = ['s-tn', 'sep', 'linea-testnet', 'sn-sep'];
 const broviderUrl = process.env.BROVIDER_URL ?? 'https://rpc.snapshot.org';
 

src/ingestor.ts

@@ -4,7 +4,6 @@ import { capture } from '@snapshot-labs/snapshot-sentry';
 import snapshot from '@snapshot-labs/snapshot.js';
 import hashTypes from '@snapshot-labs/snapshot.js/src/sign/hashedTypes.json';
 import castArray from 'lodash/castArray';
-import kebabCase from 'lodash/kebabCase';
 import { getProposal, getSpace } from './helpers/actions';
 import { isValidAlias } from './helpers/alias';
 import envelope from './helpers/envelope.json';
@@ -60,7 +59,7 @@ export default async function ingestor(req) {
     const formattedSignature = castArray(body.sig).join(',');
     const ts = Date.now() / 1e3;
     const over = 300;
-    const under = 60 * 60 * 24 * 3; // 3 days
+    const under = 60 * 60 * 24 * 6; // 6 days
     const overTs = (ts + over).toFixed();
     const underTs = (ts - under).toFixed();
     const { domain, message, types } = body.data;
@@ -160,6 +159,7 @@ export default async function ingestor(req) {
         body: message.body,
         discussion: message.discussion || '',
         choices: message.choices,
+        privacy: message.privacy || '',
         labels: message.labels || [],
         start: message.start,
         end: message.end,
@@ -168,7 +168,7 @@ export default async function ingestor(req) {
           plugins: JSON.parse(message.plugins)
         },
         type: message.type,
-        app: kebabCase(message.app || '')
+        app: message.app || ''
       };
     if (type === 'alias') payload = { alias: message.alias };
     if (type === 'statement')
@@ -186,6 +186,7 @@ export default async function ingestor(req) {
         name: message.title,
         body: message.body,
         discussion: message.discussion || '',
+        privacy: message.privacy || '',
         choices: message.choices,
         labels: message.labels || [],
         metadata: {
@@ -217,7 +218,7 @@ export default async function ingestor(req) {
         proposal: message.proposal,
         choice,
         reason: message.reason || '',
-        app: kebabCase(message.app || ''),
+        app: message.app || '',
         metadata: jsonParse(message.metadata, {})
       };
       type = 'vote';

src/writer/proposal.ts

@@ -1,7 +1,6 @@
 import { capture } from '@snapshot-labs/snapshot-sentry';
 import snapshot from '@snapshot-labs/snapshot.js';
 import networks from '@snapshot-labs/snapshot.js/src/networks.json';
-import kebabCase from 'lodash/kebabCase';
 import { validateSpaceSettings } from './settings';
 import { getSpace } from '../helpers/actions';
 import { ACTIVE_PROPOSAL_BY_AUTHOR_LIMIT, getSpaceLimits } from '../helpers/limits';
@@ -103,6 +102,10 @@ export async function verify(body): Promise<any> {
     if (msg.payload.type !== space.voting.type) return Promise.reject('invalid voting type');
   }
 
+  if (space.voting?.privacy !== 'any' && msg.payload.privacy) {
+    return Promise.reject('not allowed to set privacy');
+  }
+
   try {
     if (await isMalicious(msg.payload, space.id)) {
       return Promise.reject('invalid proposal content');
@@ -207,6 +210,10 @@ export async function action(body, ipfs, receipt, id): Promise<void> {
   const plugins = JSON.stringify(metadata.plugins || {});
   const spaceNetwork = spaceSettings.network;
   const proposalSnapshot = parseInt(msg.payload.snapshot || '0');
+  let privacy = spaceSettings.voting?.privacy || '';
+  if (privacy === 'any') {
+    privacy = msg.payload.privacy;
+  }
 
   let quorum = spaceSettings.voting?.quorum || 0;
   if (!quorum && spaceSettings.plugins?.quorum) {
@@ -238,9 +245,9 @@ export async function action(body, ipfs, receipt, id): Promise<void> {
     end: parseInt(msg.payload.end || '0'),
     quorum,
     quorum_type: (quorum && spaceSettings.voting?.quorumType) || '',
-    privacy: spaceSettings.voting?.privacy || '',
+    privacy,
     snapshot: proposalSnapshot || 0,
-    app: kebabCase(msg.payload.app || ''),
+    app: msg.payload.app,
     scores: JSON.stringify([]),
     scores_by_strategy: JSON.stringify([]),
     scores_state: 'pending',

src/writer/update-proposal.ts

@@ -49,6 +49,10 @@ export async function verify(body): Promise<any> {
 
   if (proposal.author !== body.address) return Promise.reject('Not the author');
 
+  if (space.voting?.privacy !== 'any' && msg.payload.privacy) {
+    return Promise.reject('not allowed to set privacy');
+  }
+
   const spaceUpdateError = getSpaceUpdateError({
     type: msg.payload.type,
     space
@@ -63,6 +67,11 @@ export async function action(body, ipfs): Promise<void> {
   const updated = parseInt(msg.timestamp);
   const metadata = msg.payload.metadata || {};
   const plugins = JSON.stringify(metadata.plugins || {});
+  const spaceSettings = await getSpace(msg.space);
+  let privacy = spaceSettings.voting?.privacy || '';
+  if (privacy === 'any') {
+    privacy = msg.payload.privacy;
+  }
 
   const proposal = {
     ipfs,
@@ -74,6 +83,7 @@ export async function action(body, ipfs): Promise<void> {
     discussion: msg.payload.discussion,
     choices: JSON.stringify(msg.payload.choices),
     labels: msg.payload.labels?.length ? JSON.stringify(msg.payload.labels) : null,
+    privacy,
     scores: JSON.stringify([]),
     scores_by_strategy: JSON.stringify([]),
     flagged: +containsFlaggedLinks(msg.payload.body)

src/writer/vote.ts

@@ -1,5 +1,4 @@
 import snapshot from '@snapshot-labs/snapshot.js';
-import kebabCase from 'lodash/kebabCase';
 import { getProposal } from '../helpers/actions';
 import log from '../helpers/log';
 import db from '../helpers/mysql';
@@ -108,7 +107,7 @@ export async function action(body, ipfs, receipt, id, context): Promise<void> {
   const created = parseInt(msg.timestamp);
   const choice = JSON.stringify(msg.payload.choice);
   const metadata = JSON.stringify(msg.payload.metadata || {});
-  const app = kebabCase(msg.payload.app || '');
+  const app = msg.payload.app;
   const reason = msg.payload.reason || '';
   const proposalId = msg.payload.proposal;
 

test/e2e/api.test.ts

@@ -1,8 +1,8 @@
 import fetch from 'node-fetch';
+import db from '../../src/helpers/mysql';
 import proposalInput from '../fixtures/ingestor-payload/proposal.json';
-import { spacesSqlFixtures } from '../fixtures/space';
 import proposalsFixtures from '../fixtures/proposal';
-import db from '../../src/helpers/mysql';
+import { spacesSqlFixtures } from '../fixtures/space';
 
 const HOST = `http://localhost:${process.env.PORT || 3003}`;
 const SPACE_PREFIX = 'e2e-';
@@ -71,7 +71,8 @@ describe('POST /flag', () => {
           plugins: JSON.stringify(proposal.plugins),
           choices: JSON.stringify(proposal.choices),
           scores: JSON.stringify(proposal.scores),
-          scores_by_strategy: JSON.stringify(proposal.scores_by_strategy)
+          scores_by_strategy: JSON.stringify(proposal.scores_by_strategy),
+          vp_value_by_strategy: JSON.stringify(proposal.vp_value_by_strategy || [])
         }))
         .map(async proposal => {
           db.queryAsync('INSERT INTO snapshot_sequencer_test.proposals SET ?', proposal);

test/schema.sql

@@ -28,6 +28,10 @@ CREATE TABLE spaces (
   INDEX updated (updated)
 );
 
+-- Note: The `proposals` table schema might have some discrepancies
+-- compared to the production database. This is due to legacy reasons
+-- and the challenges associated with updating the schema because of its size.
+-- `id` and `ipfs` columns should not have any default values.
 CREATE TABLE proposals (
   id VARCHAR(66) NOT NULL,
   ipfs VARCHAR(64) NOT NULL,
@@ -36,8 +40,8 @@ CREATE TABLE proposals (
   updated INT(11) DEFAULT NULL,
   space VARCHAR(64) NOT NULL,
   network VARCHAR(12) NOT NULL,
-  symbol VARCHAR(16) NOT NULL,
-  type VARCHAR(24) NOT NULL,
+  symbol VARCHAR(16) NOT NULL DEFAULT '',
+  type VARCHAR(24) NOT NULL DEFAULT '',
   strategies JSON NOT NULL,
   validation JSON NOT NULL,
   plugins JSON NOT NULL,
@@ -49,17 +53,20 @@ CREATE TABLE proposals (
   start INT(11) NOT NULL,
   end INT(11) NOT NULL,
   quorum DECIMAL(64,30) NOT NULL,
-  quorum_type VARCHAR(24) NOT NULL DEFAULT '',
+  quorum_type VARCHAR(24) DEFAULT '',
   privacy VARCHAR(24) NOT NULL,
   snapshot INT(24) NOT NULL,
   app VARCHAR(24) NOT NULL,
   scores JSON NOT NULL,
   scores_by_strategy JSON NOT NULL,
-  scores_state VARCHAR(24) NOT NULL,
+  scores_state VARCHAR(24) NOT NULL DEFAULT '',
   scores_total DECIMAL(64,30) NOT NULL,
   scores_updated INT(11) NOT NULL,
+  scores_total_value DECIMAL(64,30) NOT NULL DEFAULT '0.000000000000000000000000000000',
+  vp_value_by_strategy json NOT NULL,
   votes INT(12) NOT NULL,
   flagged INT NOT NULL DEFAULT 0,
+  cb INT NOT NULL DEFAULT 0,
   PRIMARY KEY (id),
   INDEX ipfs (ipfs),
   INDEX author (author),
@@ -73,7 +80,8 @@ CREATE TABLE proposals (
   INDEX scores_state (scores_state),
   INDEX scores_updated (scores_updated),
   INDEX votes (votes),
-  INDEX flagged (flagged)
+  INDEX flagged (flagged),
+  INDEX cb (cb)
 );
 
 CREATE TABLE votes (

yarn.lock

@@ -1350,10 +1350,10 @@
   dependencies:
     "@sentry/node" "^7.81.1"
 
-"@snapshot-labs/snapshot.js@^0.12.32":
-  version "0.12.32"
-  resolved "https://registry.yarnpkg.com/@snapshot-labs/snapshot.js/-/snapshot.js-0.12.32.tgz#a92522c8563cf9849706ba6ea5592672aba8adde"
-  integrity sha512-GRmGMjcR8qdRE94QydVQVPf93lKjJp+J/ISoSTVp3oxKrkBEzY0DZx9gFlysBqF/h69PEeykJWUOH1YfQocYHw==
+"@snapshot-labs/snapshot.js@^0.12.41":
+  version "0.12.41"
+  resolved "https://registry.yarnpkg.com/@snapshot-labs/snapshot.js/-/snapshot.js-0.12.41.tgz#3a0a2866a78f14368eb0648ef2e22a8e66422b26"
+  integrity sha512-Deikl21gUzeY/2TE8xTNsHbwTNcSqvxDnrzA1LcLwyqAzA6tigwNBlqaS5E35v7t5z7Iz62j+CPEFzxv1ta6cA==
   dependencies:
     "@ensdomains/eth-ens-namehash" "^2.0.15"
     "@ethersproject/abi" "^5.6.4"