add web sockets support for jRPC server (0xPolygonHermez#1445)

.github/workflows/test-e2e.yml

@@ -19,7 +19,7 @@ jobs:
       matrix:
         go-version: [ 1.17.x ]
         goarch: [ "amd64" ]
-        e2e-group: [ 1, 2, 3 ]
+        e2e-group: [ 1, 2, 3, 4 ]
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
@@ -58,7 +58,7 @@ jobs:
       matrix:
         go-version: [ 1.17.x ]
         goarch: [ "amd64" ]
-        e2e-group: [ 1, 2, 3 ]
+        e2e-group: [ 1, 2, 3, 4 ]
     runs-on: ubuntu-latest
     steps:
     - name: Fork based /ok-to-test checkout

Makefile

@@ -34,7 +34,7 @@ build-docker-nc: ## Builds a docker image with the node binary - but without bui
 
 .PHONY: run-rpc
 run-rpc: ## Runs all the services need to run a local zkEMV RPC node
-	docker-compose up -d zkevm-state-db zkevm-pool-db zkevm-rpc-db
+	docker-compose up -d zkevm-state-db zkevm-pool-db
 	sleep 2
 	docker-compose up -d zkevm-prover
 	sleep 5

ci/e2e-group4/shared.go

@@ -0,0 +1 @@
+../../test/e2e/shared.go

cmd/run.go

@@ -94,7 +94,6 @@ func start(cliCtx *cli.Context) error {
 			go seq.Start(ctx)
 		case RPC:
 			log.Info("Running JSON-RPC server")
-			runRPCMigrations(c.RPC.DB)
 			poolInstance := createPool(c.PoolDB, c.NetworkConfig.L2BridgeAddr, l2ChainID, st)
 			gpe := createGasPriceEstimator(c.GasPriceEstimator, st, poolInstance)
 			apis := map[string]bool{}
@@ -132,10 +131,6 @@ func runPoolMigrations(c db.Config) {
 	runMigrations(c, db.PoolMigrationName)
 }
 
-func runRPCMigrations(c db.Config) {
-	runMigrations(c, db.RPCMigrationName)
-}
-
 func runMigrations(c db.Config, name string) {
 	err := db.RunMigrationsUp(c, name)
 	if err != nil {
@@ -166,11 +161,7 @@ func runSynchronizer(cfg config.Config, etherman *etherman.Client, st *state.Sta
 }
 
 func runJSONRPCServer(c config.Config, pool *pool.Pool, st *state.State, gpe gasPriceEstimator, apis map[string]bool) {
-	storage, err := jsonrpc.NewPostgresStorage(c.RPC.DB)
-	if err != nil {
-		log.Fatal(err)
-	}
-
+	storage := jsonrpc.NewStorage()
 	c.RPC.MaxCumulativeGasUsed = c.Sequencer.MaxCumulativeGasUsed
 
 	if err := jsonrpc.NewServer(c.RPC, pool, st, gpe, storage, apis).Start(); err != nil {

config/config_test.go

@@ -301,32 +301,12 @@ func Test_Defaults(t *testing.T) {
 			expectedValue: "0x1111111111111111111111111111111111111111",
 		},
 		{
-			path:          "RPC.DB.User",
-			expectedValue: "rpc_user",
-		},
-		{
-			path:          "RPC.DB.Password",
-			expectedValue: "rpc_password",
-		},
-		{
-			path:          "RPC.DB.Name",
-			expectedValue: "rpc_db",
-		},
-		{
-			path:          "RPC.DB.Host",
-			expectedValue: "localhost",
-		},
-		{
-			path:          "RPC.DB.Port",
-			expectedValue: "5432",
-		},
-		{
-			path:          "RPC.DB.EnableLog",
+			path:          "RPC.WebSockets.Enabled",
 			expectedValue: false,
 		},
 		{
-			path:          "RPC.DB.MaxConns",
-			expectedValue: 200,
+			path:          "RPC.WebSockets.Port",
+			expectedValue: 8133,
 		},
 		{
 			path:          "Executor.URI",

config/default.go

@@ -34,7 +34,7 @@ PoEAddr = "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6"
 MaticAddr = "0x5FbDB2315678afecb367f032d93F642f64180aa3"
 GlobalExitRootManagerAddr = "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9"
 MultiGasProvider = true
-	[Etherscan]
+	[Etherman.Etherscan]
 		ApiKey = ""
 
 [EthTxManager]
@@ -56,14 +56,9 @@ MaxRequestsPerIPAndSecond = 50
 SequencerNodeURI = ""
 BroadcastURI = "127.0.0.1:61090"
 DefaultSenderAddress = "0x1111111111111111111111111111111111111111"
-	[RPC.DB]
-		User = "rpc_user"
-		Password = "rpc_password"
-		Name = "rpc_db"
-		Host = "localhost"
-		Port = "5432"
-		EnableLog = false
-		MaxConns = 200
+	[RPC.WebSockets]
+		Enabled = false
+		Port = 8133
 
 [Synchronizer]
 SyncInterval = "0s"
@@ -94,7 +89,7 @@ MaxBinaries = 262144
 MaxSteps = 8388608
 MaxAllowedFailedCounter = 50
 	[Sequencer.ProfitabilityChecker]
-		SendBatchesEvenWhenNotProfitable = "true"
+		SendBatchesEvenWhenNotProfitable = true
 
 [PriceGetter]
 Type = "default"

config/environments/local/local.node.config.toml

@@ -32,7 +32,7 @@ PoEAddr = "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6"
 MaticAddr = "0x5FbDB2315678afecb367f032d93F642f64180aa3"
 GlobalExitRootManagerAddr = "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9"
 MultiGasProvider = false
-	[Etherscan]
+	[Etherman.Etherscan]
 		ApiKey = ""
 
 [EthTxManager]
@@ -54,14 +54,9 @@ MaxRequestsPerIPAndSecond = 5000
 SequencerNodeURI = "https://internal.zkevm-test.net:2083/"
 BroadcastURI = "internal.zkevm-test.net:61090"
 DefaultSenderAddress = "0x1111111111111111111111111111111111111111"
-	[RPC.DB]
-		User = "rpc_user"
-		Password = "rpc_password"
-		Name = "rpc_db"
-		Host = "zkevm-rpc-db"
-		Port = "5432"
-		EnableLog = false
-		MaxConns = 10
+	[RPC.WebSockets]
+		Enabled = true
+		Port = 8546
 
 [Synchronizer]
 SyncInterval = "1s"

config/environments/public/public.node.config.toml

@@ -28,7 +28,7 @@ PoEAddr = "0x14cB06e8dE2222912138F9a062E5a4d9F4821409"
 MaticAddr = "0x4701Aa9471d7bfAc765D87dcb1Ea6BB23AD32733"
 GlobalExitRootManagerAddr = "0x762d09Ed110ace4EaC94acbb67b1C35D16C3297b"
 MultiGasProvider = false
-	[Etherscan]
+	[Etherman.Etherscan]
 		ApiKey = ""
 
 [RPC]
@@ -40,14 +40,9 @@ MaxRequestsPerIPAndSecond = 5000
 SequencerNodeURI = "https://public.zkevm-test.net:2083"
 BroadcastURI = "public-grpc.zkevm-test.net:61090"
 DefaultSenderAddress = "0x1111111111111111111111111111111111111111"
-	[RPC.DB]
-		User = "rpc_user"
-		Password = "rpc_password"
-		Name = "rpc_db"
-		Host = "zkevm-rpc-db"
-		Port = "5432"
-		EnableLog = false
-		MaxConns = 100
+	[RPC.WebSockets]
+		Enabled = true
+		Port = 8546
 
 [Synchronizer]
 SyncInterval = "2s"

config/readme.md

@@ -11,8 +11,6 @@ ZKEVM_NODE_DATABASE_PORT
 ZKEVM_NODE_ETHERMAN_URL
 ZKEVM_NODE_ETHERMAN_PRIVATEKEYPATH
 ZKEVM_NODE_ETHERMAN_PRIVATEKEYPASSWORD
-ZKEVM_NODE_RPC_HOST
-ZKEVM_NODE_RPC_PORT
 ZKEVM_NODE_SYNCHRONIZER_SYNCINTERVAL
 ZKEVM_NODE_SEQUENCER_INTERVALTOPROPOSEBATCH
 ZKEVM_NODE_SEQUENCER_SYNCEDBLOCKDIF

db/db.go

@@ -17,14 +17,11 @@ const (
 	StateMigrationName = "zkevm-state-db"
 	// PoolMigrationName is the name of the migration used by packr to pack the migration file
 	PoolMigrationName = "zkevm-pool-db"
-	// RPCMigrationName is the name of the migration used by packr to pack the migration file
-	RPCMigrationName = "zkevm-rpc-db"
 )
 
 var packrMigrations = map[string]*packr.Box{
 	StateMigrationName: packr.New(StateMigrationName, "./migrations/state"),
 	PoolMigrationName:  packr.New(PoolMigrationName, "./migrations/pool"),
-	RPCMigrationName:   packr.New(RPCMigrationName, "./migrations/rpc"),
 }
 
 // NewSQLDB creates a new SQL DB

db/migrations/state/0001.sql

@@ -61,6 +61,7 @@ CREATE TABLE state.l2block
     parent_hash VARCHAR,
     state_root  VARCHAR,
     received_at TIMESTAMP WITH TIME ZONE NOT NULL,
+    created_at  TIMESTAMP WITH TIME ZONE NOT NULL,
     batch_num   BIGINT                   NOT NULL REFERENCES state.batch (batch_num) ON DELETE CASCADE
 );
 

docker-compose.yml

@@ -12,7 +12,6 @@ services:
     environment:
       - ZKEVM_NODE_STATEDB_HOST=zkevm-state-db
       - ZKEVM_NODE_POOL_HOST=zkevm-pool-db
-      - ZKEVM_NODE_RPC_DB_HOST=zkevm-rpc-db
       - ZKEVM_NODE_RPC_BROADCASTURI=zkevm-broadcast:61090
     volumes:
       - ./config/environments/public/public.node.config.toml:/app/config.toml
@@ -71,23 +70,6 @@ services:
       - POSTGRES_DB=pool_db
     command: ["postgres", "-N", "500"]
 
-  zkevm-rpc-db:
-    container_name: zkevm-rpc-db
-    image: postgres
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 1G
-    ports:
-      - 5434:5432
-    environment:
-      - POSTGRES_USER=rpc_user
-      - POSTGRES_PASSWORD=rpc_password
-      - POSTGRES_DB=rpc_db
-    command: ["postgres", "-N", "500"]
-
   zkevm-prover:
     container_name: zkevm-prover
     image:  hermeznetwork/zkevm-prover:98676e8

docs/production-setup.md

@@ -184,25 +184,6 @@ services:
       - POSTGRES_PASSWORD=pool_password
       - POSTGRES_DB=pool_db
     command: ["postgres", "-N", "500"]
-
-  zkevm-rpc-db:
-    container_name: zkevm-rpc-db
-    image: postgres
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 1G
-    ports:
-      - 5434:5432
-    volumes:
-      - /$HOME/zkevm-node/.postgres-rpc:/var/lib/postgresql/data
-    environment:
-      - POSTGRES_USER=rpc_user
-      - POSTGRES_PASSWORD=rpc_password
-      - POSTGRES_DB=rpc_db
-    command: ["postgres", "-N", "500"]
 ```
 
 To run the postgres instance, go to the `zkevm-node` directory in your terminal and run the following command:
@@ -298,7 +279,6 @@ Add the following entries to the `docker-compose.yml` file
     environment:
       - ZKEVM_NODE_STATEDB_HOST=zkevm-state-db
       - ZKEVM_NODE_POOL_HOST=zkevm-pool-db
-      - ZKEVM_NODE_RPC_DB_HOST=zkevm-rpc-db
       - ZKEVM_NODE_RPC_BROADCASTURI=public-grpc.zkevm-test.net:61090
     volumes:
       - ./config.toml:/app/config.toml
@@ -384,7 +364,6 @@ services:
       environment:
         - ZKEVM_NODE_STATEDB_HOST=zkevm-state-db
         - ZKEVM_NODE_POOL_HOST=zkevm-pool-db
-        - ZKEVM_NODE_RPC_DB_HOST=zkevm-rpc-db
         - ZKEVM_NODE_RPC_PORT=8124
       volumes:
         - ./config/test.node.config.toml:/app/config.toml

jsonrpc/codec.go

@@ -49,10 +49,10 @@ type ErrorObject struct {
 }
 
 // NewResponse returns Success/Error response object
-func NewResponse(req Request, reply *[]byte, err rpcError) Response {
+func NewResponse(req Request, reply []byte, err rpcError) Response {
 	var result json.RawMessage
 	if reply != nil {
-		result = *reply
+		result = reply
 	}
 
 	var errorObj *ErrorObject
@@ -93,6 +93,30 @@ func (r Response) MarshalJSON() ([]byte, error) {
 	})
 }
 
+// Bytes return the serialized response
+func (s Response) Bytes() ([]byte, error) {
+	return json.Marshal(s)
+}
+
+// SubscriptionResponse used to push response for filters
+// that have an active web socket connection
+type SubscriptionResponse struct {
+	JSONRPC string                     `json:"jsonrpc"`
+	Method  string                     `json:"method"`
+	Params  SubscriptionResponseParams `json:"params"`
+}
+
+// SubscriptionResponseParams parameters for subscription responses
+type SubscriptionResponseParams struct {
+	Subscription string          `json:"subscription"`
+	Result       json.RawMessage `json:"result"`
+}
+
+// Bytes return the serialized response
+func (s SubscriptionResponse) Bytes() ([]byte, error) {
+	return json.Marshal(s)
+}
+
 // BlockNumber is the number of a ethereum block
 type BlockNumber int64