ethereum
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -1,6 +1,5 @@
 name: node-crawler workflow
 
-
 on:
   push:
     branches:
@@ -16,26 +15,26 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.21.4
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v4
       with:
-        go-version: 1.20.5
-    - name: Download golangci-lint
-      run: wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s latest
-    - name: Lint
-      run: ./bin/golangci-lint run --config .golangci.yml
+        version: v1.56.2
     - name: Vet
       run: go vet ./...
 
   test:
     runs-on: ubuntu-latest
     needs: lint
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v5
       with:
-        go-version: 1.20.5
+        go-version: 1.21.4
     - name: Test
-      run: go test -v ./...
+      run: go test -v ./...
diff --git a/.gitignore b/.gitignore
@@ -16,11 +16,19 @@ build
 
 npm-debug.log*
 
-nodetable
-nodes
-node-crawler-backend.exe
-node-crawler-backend
+# application
 /data/
+/snapshots/
+/*.db
+/*.db.gz
+/*.db-*
+node-crawler-backend
+node-crawler-backend.exe
+*.keys
+nodes
+nodetable
+public/*_templ.go
 
 # nix
 result
+.data
diff --git a/.golangci.yml b/.golangci.yml
@@ -18,14 +18,16 @@ linters:
     - misspell
     - unconvert
     - typecheck
+    - exhaustruct
 # this repo contains a few copied files from go-ethereum, 
 # and some of them have unused fields/functions
-    #- unused
+    # - unused
     - staticcheck
     - bidichk
     - durationcheck
     - exportloopref
     - whitespace
+    - promlinter
 
     # - structcheck # lots of false positives
     # - errcheck #lot of false positives

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Compile api
-FROM golang:1.20-alpine AS builder
+FROM golang:1.21-alpine AS builder
 WORKDIR /app
 
 COPY go.mod go.sum ./
@@ -10,7 +10,7 @@ RUN go build ./cmd/crawler
 
 
 # Copy compiled stuff and run it
-FROM golang:1.20-alpine
+FROM alpine
 
 COPY --from=builder /app/crawler /app/crawler
 

diff --git a/README.md b/README.md
@@ -93,9 +93,9 @@ go run ./cmd/crawler
 - golang
 - sqlite3
 
-##### Country location
+##### City location
 
-- `GeoLite2-Country.mmdb` file from [https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en)
+- `GeoLite2-City.mmdb` file from [https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data?lang=en)
   - you will have to create an account to get access to this file
 
 #### Development
@@ -119,7 +119,7 @@ go build ./cmd/crawler -o /usr/bin/node-crawler
 ```
 
 Create a systemd service similarly to above API example. In executed command, override default settings by pointing crawler database to chosen path and setting period to write crawled nodes.
-If you want to get the country that a Node is in you have to specify the location the geoIP database as well.
+If you want to get the city that a Node is in you have to specify the location the geoIP database as well.
 
 ##### No GeoIP
 
@@ -130,7 +130,7 @@ node-crawler crawl --timeout 10m --crawler-db /path/to/database
 ##### With GeoIP
 
 ```
-node-crawler crawl --timeout 10m --crawler /path/to/database --geoipdb GeoLite2-Country.mmdb
+node-crawler crawl --timeout 10m --crawler /path/to/database --geoipdb GeoLite2-City.mmdb
 ```
 
 ### Docker setup
@@ -240,19 +240,20 @@ Your example `configuration.nix`:
     nodeCrawler = {
       enable = true;
       hostName = "server hostname";
+      api.enodePubkey = "asdf1234...";
       nginx = {
         forceSSL = true;
         enableACME = true;
       };
     };
 
-    # Needed for the node crawler to get the country
+    # Needed for the node crawler to get the city
     # of the crawled IP address.
     geoipupdate = {
       enable = true;
       settings = {
         EditionIDs = [
-          "GeoLite2-Country"
+          "GeoLite2-City"
         ];
         AccountID = account_id;
         LicenseKey = "location of licence key on server";
@@ -268,3 +269,17 @@ Your example `configuration.nix`:
   };
 }
 ```
+
+### TODO
+- [ ] Enums instead of numbers in the URLs
+- [ ] More stats
+  - [ ] Client Versions
+  - [ ] Countries link to show cities in that country
+- [ ] More filters
+  - [ ] Country/City
+  - [ ] OS/Arch
+- [ ] Custom inputs for Network ID filter
+- [ ] Info/help where more details could be useful
+- [ ] Expand help page
+  - [ ] What do the error messages mean, what should the user do for each one?
+  - [ ] Instructions on how to connect for each client
diff --git a/cmd/crawler/api.go b/cmd/crawler/api.go
@@ -1,149 +1,88 @@
 package main
 
 import (
-	"database/sql"
 	"fmt"
-	"os"
+	"net/http"
 	"sync"
-	"time"
 
 	_ "modernc.org/sqlite"
 
-	"github.com/ethereum/go-ethereum/log"
+	"log/slog"
+
 	"github.com/ethereum/node-crawler/pkg/api"
-	"github.com/ethereum/node-crawler/pkg/apidb"
-	"github.com/ethereum/node-crawler/pkg/crawlerdb"
+	"github.com/ethereum/node-crawler/pkg/common"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/urfave/cli/v2"
 )
 
-var (
-	apiCommand = &cli.Command{
-		Name:   "api",
-		Usage:  "API server for the crawler",
-		Action: startAPI,
-		Flags: []cli.Flag{
-			&apiDBFlag,
-			&apiListenAddrFlag,
-			&autovacuumFlag,
-			&busyTimeoutFlag,
-			&crawlerDBFlag,
-			&dropNodesTimeFlag,
-		},
+//nolint:exhaustruct
+var apiCommand = &cli.Command{
+	Name:   "api",
+	Usage:  "API server for the crawler",
+	Action: runAPI,
+	Flags: []cli.Flag{
+		&apiListenAddrFlag,
+		&busyTimeoutFlag,
+		&dropNodesTimeFlag,
+		&enodeAddrFlag,
+		&listenStartPortFlag,
+		&metricsAddressFlag,
+		&executionNodeKeysFileFlag,
+		&postgresFlag,
+		&snapshotDirFlag,
+		&statsUpdateFrequencyFlag,
+	},
+}
+
+func runAPI(cCtx *cli.Context) error {
+	db, err := openDBReader(cCtx)
+	if err != nil {
+		return fmt.Errorf("open db failed: %w", err)
 	}
-)
+	defer db.Close()
 
-func startAPI(ctx *cli.Context) error {
-	autovacuum := ctx.String(autovacuumFlag.Name)
-	busyTimeout := ctx.Uint64(busyTimeoutFlag.Name)
+	wg := new(sync.WaitGroup)
+	wg.Add(1)
 
-	crawlerDB, err := openSQLiteDB(
-		ctx.String(crawlerDBFlag.Name),
-		autovacuum,
-		busyTimeout,
-	)
+	enodes, err := readEnodes(cCtx)
 	if err != nil {
-		return err
+		return fmt.Errorf("Read enodes failed: %w", err)
 	}
 
-	apiDBPath := ctx.String(apiDBFlag.Name)
-	shouldInit := false
-	if _, err := os.Stat(apiDBPath); os.IsNotExist(err) {
-		shouldInit = true
-	}
-	nodeDB, err := openSQLiteDB(
-		apiDBPath,
-		autovacuum,
-		busyTimeout,
+	// Start the API deamon
+	api := api.New(
+		db,
+		statsUpdateFrequencyFlag.Get(cCtx),
+		enodes,
+		snapshotDirFlag.Get(cCtx),
+	)
+	go api.StartServer(
+		wg,
+		apiListenAddrFlag.Get(cCtx),
 	)
-	if err != nil {
-		return err
-	}
-	if shouldInit {
-		log.Info("DB did not exist, init")
-		if err := apidb.CreateDB(nodeDB); err != nil {
-			return err
-		}
-	}
-	var wg sync.WaitGroup
-	wg.Add(3)
 
-	// Start reading deamon
-	go newNodeDeamon(&wg, crawlerDB, nodeDB)
-	go dropDeamon(&wg, nodeDB, ctx.Duration(dropNodesTimeFlag.Name))
+	// Start metrics server
+	metricsAddr := metricsAddressFlag.Get(cCtx)
+	slog.Info("starting metrics server", "address", metricsAddr)
+	http.Handle("/metrics", promhttp.Handler())
+	http.ListenAndServe(metricsAddr, nil)
 
-	// Start the API deamon
-	apiAddress := ctx.String(apiListenAddrFlag.Name)
-	apiDeamon := api.New(apiAddress, nodeDB)
-	go apiDeamon.HandleRequests(&wg)
 	wg.Wait()
 
 	return nil
 }
 
-func transferNewNodes(crawlerDB, nodeDB *sql.DB) error {
-	crawlerDBTx, err := crawlerDB.Begin()
+func readEnodes(cCtx *cli.Context) ([]string, error) {
+	keys, err := common.ReadNodeKeys(executionNodeKeysFileFlag.Get(cCtx))
 	if err != nil {
-		// Sometimes error occur trying to read the crawler database, but
-		// they are normally recoverable, and a lot of the time, it's
-		// because the database is locked by the crawler.
-		return fmt.Errorf("error starting transaction to read nodes: %w", err)
+		return nil, fmt.Errorf("read node keys: %w", err)
 	}
-	defer crawlerDBTx.Rollback()
 
-	nodes, err := crawlerdb.ReadAndDeleteUnseenNodes(crawlerDBTx)
-	if err != nil {
-		// Simiar to nodeDB.Begin() error
-		return fmt.Errorf("error reading nodes: %w", err)
-	}
-
-	if len(nodes) > 0 {
-		err := apidb.InsertCrawledNodes(nodeDB, nodes)
-		if err != nil {
-			// This shouldn't happen because the database is not shared in this
-			// instance, so there shouldn't be lock errors, but anything can
-			// happen. We will still try again.
-			return fmt.Errorf("error inserting nodes: %w", err)
-		}
-		log.Info("Nodes inserted", "len", len(nodes))
-	}
-
-	crawlerDBTx.Commit()
-	return nil
-}
-
-// newNodeDeamon reads new nodes from the crawler and puts them in the db
-// Might trigger the invalidation of caches for the api in the future
-func newNodeDeamon(wg *sync.WaitGroup, crawlerDB, nodeDB *sql.DB) {
-	defer wg.Done()
-
-	// This is so that we can make some kind of exponential backoff for the
-	// retries.
-	retryTimeout := time.Minute
-
-	for {
-		err := transferNewNodes(crawlerDB, nodeDB)
-		if err != nil {
-			log.Error("Failure in transferring new nodes", "err", err)
-			time.Sleep(retryTimeout)
-			retryTimeout *= 2
-			continue
-		}
-
-		retryTimeout = time.Minute
-		time.Sleep(time.Second)
-	}
-}
-
-func dropDeamon(wg *sync.WaitGroup, db *sql.DB, dropTimeout time.Duration) {
-	defer wg.Done()
-	ticker := time.NewTicker(10 * time.Minute)
-	defer ticker.Stop()
+	enodes := common.KeysToEnodes(
+		keys,
+		enodeAddrFlag.Get(cCtx),
+		listenStartPortFlag.Get(cCtx),
+	)
 
-	for {
-		<-ticker.C
-		err := apidb.DropOldNodes(db, dropTimeout)
-		if err != nil {
-			panic(err)
-		}
-	}
+	return enodes, nil
 }