Feat/create metrics module to monitor transeptor (#50)

* feat: init metrics package

* feat: add docs to setuo Grafana

* feat: add metrics server running on a different port. Update readme to show new metric command line args. Add nx cache to speed up local dev build time

* feat: track CPU and memory every second, then store in influxdb

* feat: add work thread to collect and store system metrics

* chore: update metrics readme to walkthrough adding data sources to Grafana

* chore: update readme to add metrics feature

* feat: add v1 traseptor metrics  dashboard

* chore: release 0.5.0-alpha.0

* chore: update readme

.dockerignore

@@ -7,6 +7,12 @@ compliance_test.sh
 # Docker
 Dockerfile
 
+# Metrics data
+/influxdb-data
+/grafana-data
+/packages/metrics/screen-shoots/
+/packages/metrics/dashboards/
+
 # secrets
 .env
 

.env.sample

@@ -4,3 +4,4 @@ ALCHEMY_API_KEY=<your-alcemy-api-key>
 BENEFICIARY=<address_to_receive_funds>
 WHITE_LIST=<address_to_whitelist_SEPARATEDBY_COMMA>
 BLACK_LIST=<address_to_blacklist_SEPARATEDBY_COMMA>
+INFLUX_TOKEN=DEV_TOKEN

.github/workflows/bundler-spec-test.yml

@@ -93,4 +93,4 @@ jobs:
 
       - name: Run tests
         working-directory: ./bundler-spec-tests
-        run: pdm run pytest -rA -W ignore::DeprecationWarning --url http://127.0.0.1:3000/rpc/ --entry-point 0x5FF137D4b0FDCD49DcA30c7CF57E578a026d2789 --ethereum-node http://localhost:8545
+        run: pdm run pytest -rA -W ignore::DeprecationWarning --url http://127.0.0.1:4000/rpc/ --entry-point 0x5FF137D4b0FDCD49DcA30c7CF57E578a026d2789 --ethereum-node http://localhost:8545

.gitignore

@@ -27,3 +27,8 @@ packages/tools/typechain-types
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# Metrics data
+/influxdb-data
+/grafana-data
+

README.md

@@ -24,7 +24,6 @@
 > :warning: **This repository is currently under active development.**
 > See our road-map [here](https://hackmd.io/@V00D00-child/SyXKL6Kmn#Project-StatusRoadmap-)
 
-
 ## 🚀 Development
 
 Everything you need to get started developing with Transeptor.
@@ -47,7 +46,7 @@ Open a new terminal and then start the bundler node.
 npm run start:dev
 ```
 
-The bundler will start on `http://localhost:3000/rpc`.
+The bundler will start on `http://localhost:3001/rpc`.
 
 ### Test
 
@@ -67,14 +66,15 @@ or
 npm run lint:fix
 ```
 
-## 🔧  Run from source
-1. Install dependencies `npm install`
-2. Build all packages `npm run build`
-2. Give the script execution permission `chmod +x ./transeptor`
+## 🔧 Run from source
+1. Use correct node version `nvm use`
+2. Install dependencies `npm install`
+3. Build all packages `npm run build`
+4. Copy values in `.env.sample` into `.env` and fill in the values with your own.
+5. Give the script execution permission `chmod +x ./transeptor`
 
 Now let's start the bundler node.(make sure to pass your command line arguments)
 ```bash
-
 ./transeptor --httpApi web3,eth,debug --txMode base
 ```
 
@@ -90,9 +90,13 @@ Run image locally
 npm run start:bundler-docker
 ```
 
+## ✨ Features
+- **Full Validation** - Uses geth `debug_traceCall` method to enforce the full spec storage access rules and opcode banning. 
+- **Partial Validation** - Standard call to entry Point Contract `simulateValidation()`. No storage access rules and opcode banning. User `--unsafe` flag to enable.
+- **Metrics** - Metrics gives insight into the bundler node to allow for performance tuning and debugging. Transeptor bundler can be be configure to store metrics using a push(InfluxDB) and pull(Prometheus) metrics system. Grafana is used to visualize all the metrics. Use `--metrics` flag to enable.
+
+
 ## 🧮 Command line arguments
-- **Full Validation** - Uses geth `debug_traceCall` method to enforce the full spec storage access rules and opcode banning.
-- **Partial Validation** - Standard call to entry Point Contract `simulateValidation()`. No storage access rules and opcode banning.
 
 List of all command line arguments supported by the bundler.
 
@@ -106,13 +110,18 @@ List of all command line arguments supported by the bundler.
 |        `--auto`        | `boolean` | automatic bundling                                                  | `false`                 |
 | `--autoBundleInterval` | `number`  | auto bundler interval in (ms)                                       | `12000`                |
 |     `--bundleSize`     | `number`  | maximum # of pending mempool entities                               | `10`                    |
-|        `--port`        | `number`  | server listening port                                               | `3000`                  |
+|        `--port`        | `number`  | server listening port                                               | `4000`                  |
 |  `--minUnstakeDelay`   | `number`  | time paymaster has to wait to unlock the stake (seconds)            | `0`                     |
 |      `--minStake`      | `number`  | minimum stake an entity has to have to pass the reputation system\* | `1`                     |
 |       `--txMode`       | `string`  | bundler transaction mode (base, conditional, searcher)              | `base`                  |
 |       `--unsafe`       | `boolean` | UNSAFE mode: no storage or opcode checks \*\*                       | `false`                 |
 |        `--p2p`         | `boolean` | enable p2p mode enabled(under development)                                                    | `false`                 |
 |        `--findPeers`         | `boolean` | search for peers when p2p enabled(under development)                                                  | `false`                 |
+|       `--metrics`       | `boolean`  | bundler metrics enabled              | `false`                  |
+|       `--metricsPort`       | `number`  | metrics server listening port              | `4001`                  |
+|       `--influxdbUrl`       | `string`  | url influxdb is running on            | `http://localhost:8086'`                  |
+|       `--influxdbOrg`       | `string`  | influxdb org              | `transeptor-labs`                  |
+|       `--influxdbBucket`       | `string`  | influxdb bucket              | `transeptor_metrics`                  |
 
 \*When staked, an entity is also allowed to use its own associated storage, in addition to senders associated storage as ETH.
 **safe mode requires debug_traceCall support on eth node. Only base and conditional txMode are supported in safe mode. \***0x5FF137D4b0FDCD49DcA30c7CF57E578a026d2789

lerna.json

@@ -1,4 +1,4 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "0.4.0-alpha"
+  "version": "0.5.0-alpha.0"
 }

nx.json

@@ -0,0 +1,10 @@
+{
+    "tasksRunnerOptions": {
+      "default": {
+        "runner": "nx/tasks-runners/default",
+        "options": {
+          "cacheableOperations": ["build"]
+        }
+      }
+    }
+  }

package.json

@@ -23,15 +23,19 @@
     "clean:deps": "rm -rf ./node_modules ./packages/*/node_modules",
     "clean:coverage": "rm -rf ./coverage ./packages/*/coverage",
     "lerna-clean": "npx lerna run clean",
-    "build": "npx lerna run build --concurrency 3",
-    "geth-node": "cd packages/tools && npm run geth:start",
-    "bundler-prep": "cd packages/tools && npm run bundler-prep",
+    "build": "cd packages/types && npm run build && npx lerna run build --concurrency 3",
     "watch:dev": "npx lerna watch -- lerna run build",
-    "start:dev": "test -n \"$(find packages/*/lib -type f)\" || npm run build && node packages/cli/lib/src/cli.js --httpApi web3,eth,debug --txMode base",
+    "start:dev": "test -n \"$(find packages/*/lib -type f)\" || npm run build && node packages/cli/lib/src/cli.js --httpApi web3,eth,debug --txMode base | pino-pretty",
+    "start:dev-metrics": "test -n \"$(find packages/*/lib -type f)\" || npm run build && node packages/cli/lib/src/cli.js --httpApi web3,eth,debug --txMode base --metrics | pino-pretty",
     "test": "npx lerna run test --concurrency 2",
     "lint": "npx lerna run lint --concurrency 2",
     "lint:fix": "npx lerna run lint:fix --concurrency 2",
-    "version": "npx lerna version --no-private"
+    "version": "npx lerna version --no-private",
+    "///cK": "________________________DOCKER CONTAINERS______________________________",
+    "geth-node": "cd packages/tools && npm run geth:start",
+    "bundler-prep": "cd packages/tools && npm run bundler-prep",
+    "influxdb": "cd packages/tools && npm run start:influxdb",
+    "grafana": "cd packages/tools && npm run start:grafana"
   },
   "devDependencies": {
     "@account-abstraction/contracts": "^0.6.0",