docs

.env.example

@@ -1,3 +1,5 @@
+PG_PASSWORD=
+READONLY_PASSWORD=
 DB_PORT=23798
 DASHBOARD_PORT=8501
 LEGACY_DB_LOCATION=

Makefile

@@ -1,5 +1,19 @@
+include .env
+export
+
 .PHONY: build wrap dbt
 
+DB_NAME ?= base_goerli
+CONTAINER_NAME ?= data-db-1
+
+# Target to recreate the database
+recreate-db:
+	docker exec -it $(CONTAINER_NAME) /bin/bash -c "PGPASSWORD=${PG_PASSWORD} psql -U postgres -c 'DROP DATABASE IF EXISTS $(DB_NAME);'"
+	docker exec -it $(CONTAINER_NAME) /bin/bash -c "PGPASSWORD=${PG_PASSWORD} psql -U postgres -c 'CREATE DATABASE $(DB_NAME);'"
+
+reset-pw:
+	docker exec -it $(CONTAINER_NAME) /bin/bash -c "PGPASSWORD=$(PG_PASSWORD) psql -U postgres -c 'ALTER USER analytics WITH PASSWORD '\''$(READONLY_PASSWORD)'\'';'"
+
 build:
 	docker compose build transformer
 

README.md

@@ -8,3 +8,37 @@ A collection of services to index, store, and transform data related to Syntheti
 * [**Indexers**](./indexers/) - Blockchain indexers using Subsquid archives to index Synthetix smart contracts. These indexers are used to populate a Postgres database with raw event log data.
 * [**Transformers**](./transformers/) - Services that transform raw event log data into a format that is more useful for querying. These services are used to populate a Postgres database with transformed data using [dbt](https://www.getdbt.com/).
 * [**Dashboard**](./dashboard/) - A collection of dashboards built using [streamlit](https://streamlit.io/) and connected directly to the Postgres database.
+
+For more information about these services, visit the README in the respective directory.
+
+## Usage
+
+### Configuration
+
+The services are all managed using [docker compose](https://docs.docker.com/compose/). Review the `docker-compose.yml` file to view the services that will run on startup. Some of these services require configuration through environment variables, which should be copied and populated in a `.env` file. In the root directory use these to configure your environment:
+
+- `PG_PASSWORD`: The password for the admin user of the Postgres database.
+- `READONLY_PASSWORD`: The password for a configured read-only user, used for dashboards. Change this password and run `make reset-pw` to update the user's password.
+- `DB_PORT`: The port that will be used to expose the Postgres database. If left blank, the database will only be exposed to the docker network.
+- `DASHBOARD_PORT`: The port used to expose the streamlit dashboards.
+- `LEGACY_DB_LOCATION`: The location of a legacy SQLite database. This database is used in Optimism Mainnet dashboards summarizing activity on the legacy perps V2 contracts.
+
+Ensure that you also configure the environment variable for each of the indexers (ex. `./indexers/base-mainnet/.env`)
+
+### Start Indexers
+
+Once you have configured your environment, run `docker compose up -d --build` to build and run the services in detached mode. By default, the service will start a Postgres database, indexers for each network, and a streamlit dashboard on startup. Each indexer will write data to a database corresponding with the network name (ex. `base_mainnet`). You can view the logs for each service using `docker compose logs -f <service-name>`.
+
+The dashboard service relies on transformed data in the `analytics` database. To populate this database, you must run the transformers.
+
+### Running Transformers
+
+To simplify queries and transformed data, you must run the transformers to populate the `analytics` database. This happens in two steps, first by wrapping the raw tables as foreigns tables in the `analytics` database, then running dbt for each of the relevant schemas. To do this, run:
+
+```bash
+make build
+make wrap
+make dbt
+```
+
+You should see output confirming that dbt has run for each network, and created a set of tables and views in the `analytics` database. The running dashboard service will automatically detect these tables and views and populate the dashboards with data. To view the dashboards, visit `http://localhost:<DASHBOARD_PORT>` in your browser.

dashboard/README.md

@@ -0,0 +1,15 @@
+# Dashboard
+
+The dashboard service provides a suite of dashboards using [streamlit](https://streamlit.io/). The dashboards are designed to use the data marts created by the [transformers](../transformers/) service. 
+
+## Usage
+
+The service runs as a docker container, and is started along with the other services in the `docker compose` setup. The dashboard is available at the port set by the `DASHBOARD_PORT` environment variable, which defaults to `8501`.
+
+## Development
+
+The dashboard service consists of a page in the `pages` directory which corresponds to a network (OP mainnet, Base Goerli, Base Mainnet, etc.) Each page is set up to import a set of modules from the `modules` directory. To add new modules:
+
+1. Create a new python file in the `modules` directory for the corresponding network
+1. Import the module in the `pages` file for the corresponding network
+1. Add the module to the `pages` dictionary for that page

dashboard/utils/data.py

@@ -1,12 +1,16 @@
+import os
 import streamlit as st
 import sqlalchemy
 import pandas as pd
 
+# get the password from the environment
+DB_PASS = os.environ.get("DB_PASS")
+
 # Database connection parameters
 DEFAULT_DB_CONFIG = {
     "dbname": "analytics",
     "user": "analytics",
-    "password": "analytics",
+    "password": DB_PASS,
     "host": "db",
     "port": 5432,
 }

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     networks:
       - data
     environment:
-      POSTGRES_PASSWORD: postgres
+      POSTGRES_PASSWORD: $PG_PASSWORD
     volumes:
       - ./postgres:/docker-entrypoint-initdb.d
       - ./postgres-data:/var/lib/postgresql/data
@@ -23,6 +23,8 @@ services:
     depends_on:
       - db
     restart: always
+    environment:
+      DB_PASS: $PG_PASSWORD
     env_file:
       - ./indexers/base-mainnet/.env
 
@@ -34,6 +36,8 @@ services:
     depends_on:
       - db
     restart: always
+    environment:
+      DB_PASS: $PG_PASSWORD
     env_file:
       - ./indexers/base-goerli/.env
 
@@ -45,6 +49,8 @@ services:
     depends_on:
       - db
     restart: always
+    environment:
+      DB_PASS: $PG_PASSWORD
     env_file:
       - ./indexers/optimism-goerli/.env
 
@@ -56,6 +62,8 @@ services:
     depends_on:
       - db
     restart: always
+    environment:
+      DB_PASS: $PG_PASSWORD
     env_file:
       - ./indexers/optimism-mainnet/.env
 
@@ -64,6 +72,8 @@ services:
       context: ./transformers
     depends_on:
       - db
+    environment:
+      PG_PASSWORD: $PG_PASSWORD
     networks:
       - data
 
@@ -80,6 +90,8 @@ services:
     volumes:
       - ${LEGACY_DB_LOCATION}:/app/data/perps.db
       # - ./dashboard:/app # uncomment to enable dashboard development
+    environment:
+      DB_PASS: $READONLY_PASSWORD
     deploy:
       resources:
         limits:

indexers/README.md

@@ -1,14 +1,12 @@
 # Indexers
 
-This service contains a set of blockchain indexers that use Subsquid archives to index Synthetix smart contracts. A fully configured docker compose file is provided to run the indexers and a Postgres database. 
+This service contains a set of blockchain indexers that use Subsquid archives and public RPCs to index Synthetix smart contracts. A fully configured docker compose file is provided to run the indexers and a Postgres database.
 
 ## Getting Started
 
 1. Ensure you have installed Docker with Docker Compose. If you have not, follow the instructions [here](https://docs.docker.com/compose/install/).
-1. Clone this repository.
-1. Copy the `indexers/.env.example` file to `indexers/.env`. This file contains an environment variable for Postgres database port.
-1. Each network has a directory in the `indexers` directory (ie `optimism_mainnet`). Navigate to each network and copy the `.env.example` file to `.env`, filling in any additional settings. The example contains defaults which will work for most users, however the optional `RPC_ENDPOINT` value can be set to a custom RPC endpoint if you would like to index directly from a node.
-1. Run `docker-compose up` from the root directory of this repository. This will start the indexers and a Postgres database. The indexers will begin indexing the specified network and will populate the database with raw event log data.
+1. Each network has a directory in the `indexers` directory (ie `optimism-mainnet`). Navigate to each network and copy the `.env.example` file to `.env`, filling in any additional settings. The example contains defaults which will work for most users, however the optional `RPC_ENDPOINT` value can be set to a custom RPC endpoint if you would like to index directly from a node.
+1. Run `docker compose up` from the root directory of this repository. This will start the indexers and a Postgres database. The indexers will begin indexing the specified network and will populate the database with raw event log data.
 
 ## Querying Data
 
@@ -18,9 +16,10 @@ A Postgres database will be running in a container. You can query the database u
 Host: localhost
 Port: 23798
 Username: postgres
-Password: postgres
+Password: $PG_PASSWORD (from .env)
 ```
 
 Notes:
-* The service will have one database for each network. For example, all Optimism Mainnet events can be found in the `optimism-mainnet` database.
+* The service will have one database for each network. For example, all Optimism Mainnet events can be found in the `optimism_mainnet` database.
 * Database tables for events follow the format `{contract_name}_event_{event_name}`, for example: `perps_market_proxy_event_account_created`.
+* By default an `analytics` database exists, but will not be populated with data. To populate this database with transformed data, see the [transformers](../transformers/) directory.

transformers/README.md

@@ -0,0 +1,30 @@
+# Transformers
+
+This services makes use of [dbt](https://www.getdbt.com/) to transform raw event log data into a format that is more useful for analytics and dashboards. The service is configured to run dbt for each of the networks that are indexed by the [indexers](../indexers/) service. The service will run dbt for each of the networks, and will create a set of tables and views in the `analytics` database.
+
+## Usage
+
+The most simple interface for running transformations is using `make` commands. See the [Makefile](../Makefile) in the root directory for the commands. The following commands are available:
+```bash
+## dbt operations
+# build the transformations docker image
+make build
+
+# run dbt for all networks
+make dbt
+
+## Postgres operations
+# wrap the raw tables from the indexers as foreign tables in the analytics database
+make wrap
+
+# drop and recreate a database
+# this can be useful if you want to index from scratch
+make recreate-db DB=base_goerli
+
+# reset the read-only user password
+make reset-pw
+```
+
+## Models
+
+Models are split into two categories: `raw` and `marts`. The `raw` models load event logs directly from the indexers without any transformations. Models in the `marts` directory combine and transform data from the `raw` models into a format that is more useful for analytics and dashboards. The `marts` models are the ones that are used by the [dashboards](../dashboards/) service.

transformers/synthetix/profiles/profiles.yml

@@ -5,7 +5,7 @@ dev:
       type: postgres
       host: localhost
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 23798
       dbname: analytics
       schema: base_goerli
@@ -18,7 +18,7 @@ dev:
       type: postgres
       host: localhost
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 23798
       dbname: analytics
       schema: optimism_goerli
@@ -31,7 +31,7 @@ dev:
       type: postgres
       host: localhost
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 23798
       dbname: analytics
       schema: optimism_mainnet
@@ -44,7 +44,7 @@ dev:
       type: postgres
       host: localhost
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 23798
       dbname: analytics
       schema: base_mainnet
@@ -60,7 +60,7 @@ docker:
       type: postgres
       host: db
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 5432
       dbname: analytics
       schema: base_goerli
@@ -73,7 +73,7 @@ docker:
       type: postgres
       host: db
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 5432
       dbname: analytics
       schema: optimism_goerli
@@ -86,7 +86,7 @@ docker:
       type: postgres
       host: db
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 5432
       dbname: analytics
       schema: optimism_mainnet
@@ -99,7 +99,7 @@ docker:
       type: postgres
       host: db
       user: postgres
-      password: postgres
+      password: "{{ env_var('PG_PASSWORD') }}"
       port: 5432
       dbname: analytics
       schema: base_mainnet

transformers/synthetix/scripts/wrap_tables.py

@@ -1,5 +1,8 @@
+import os
 import psycopg2
 
+PG_PASSWORD = os.environ["PG_PASSWORD"]
+
 
 def setup_fdw(cursor, server_name, local_db_params, remote_db_params):
     # Create the postgres_fdw extension if it doesn't exist
@@ -104,7 +107,7 @@ def create_foreign_tables(database_name, db_params, source_schema="public"):
 
 
 # Database connection parameters for the local database
-db_params = {"host": "db", "port": 5432, "user": "postgres", "password": "postgres"}
+db_params = {"host": "db", "port": 5432, "user": "postgres", "password": PG_PASSWORD}
 
 create_foreign_tables("base_goerli", db_params)
 create_foreign_tables("optimism_goerli", db_params)