Skip to content

Commit

Permalink
Merge pull request #40 from dbecorp/metrics-prefixes
Browse files Browse the repository at this point in the history
Prefix metric names with package name and custom prefixes
  • Loading branch information
jakthom authored Oct 24, 2024
2 parents bdd83b0 + 99229f0 commit 6ef3ce7
Show file tree
Hide file tree
Showing 16 changed files with 182 additions and 115 deletions.
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@
<img src="assets/heracles.png" width="250" align="right"/>


Hercules is a DuckDB-powered Prometheus exporter that supercharges metrics.
### Hercules is a Prometheus-compatible exporter that supercharges your metrics.


**Generate prometheus-compatible metrics** from parquet, csv files, json logs, data lakes, databases, http endpoints, and much more.
* **Generate prometheus-compatible metrics** from parquet, csv files, json logs, data lakes, databases, http endpoints, and much more.

**Generate enriched, labeled** metrics properly from the source; don't relabel using your favorite metrics database.
* **Generate enriched, labeled** metrics properly from the source; don't relabel using your favorite metrics database.

**Embrace** the pantheon of metrics harvesting using Prometheus-compatible scrape targets that easily tame [TPC-H benchmarks](https://www.tpc.org/information/benchmarks5.asp).
* **Embrace** the pantheon of metrics harvesting using Prometheus-compatible scrape targets that easily tame [TPC-H benchmarks](https://www.tpc.org/information/benchmarks5.asp).


# Getting Started
Expand Down
36 changes: 22 additions & 14 deletions cmd/hercules/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@ import (
"github.com/dbecorp/hercules/pkg/config"
"github.com/dbecorp/hercules/pkg/flock"
herculespackage "github.com/dbecorp/hercules/pkg/herculesPackage"
metrics "github.com/dbecorp/hercules/pkg/metrics"
registry "github.com/dbecorp/hercules/pkg/metricRegistry"
"github.com/dbecorp/hercules/pkg/middleware"
herculestypes "github.com/dbecorp/hercules/pkg/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
Expand All @@ -25,12 +26,12 @@ import (
var VERSION string

type Hercules struct {
config config.Config
db *sql.DB
packages []herculespackage.Package
conn *sql.Conn
metricRegistry *metrics.MetricRegistry
debug bool
config config.Config
db *sql.DB
packages []herculespackage.Package
conn *sql.Conn
metricRegistries []*registry.MetricRegistry
debug bool
}

func (d *Hercules) configure() {
Expand Down Expand Up @@ -80,13 +81,20 @@ func (d *Hercules) initializePackages() {
}
}

func (d *Hercules) initializeRegistry() {
// Merge metric definitions from all packages
metricDefinitions := metrics.MetricDefinitions{}
func (d *Hercules) initializeRegistries() {
// Register a registry for each package
for _, pkg := range d.packages {
metricDefinitions.Merge(pkg.Metrics)
metricMetadata := herculestypes.MetricMetadata{
PackageName: pkg.Name,
MetricPrefix: pkg.MetricPrefix,
Labels: d.config.InstanceLabels(),
}
if d.metricRegistries == nil {
d.metricRegistries = []*registry.MetricRegistry{registry.NewMetricRegistry(pkg.Metrics, metricMetadata)}
} else {
d.metricRegistries = append(d.metricRegistries, registry.NewMetricRegistry(pkg.Metrics, metricMetadata))
}
}
d.metricRegistry = metrics.NewMetricRegistry(metricDefinitions, d.config.InstanceLabels())
}

func (d *Hercules) Initialize() {
Expand All @@ -95,14 +103,14 @@ func (d *Hercules) Initialize() {
d.initializeFlock()
d.loadPackages()
d.initializePackages()
d.initializeRegistry()
d.initializeRegistries()
log.Debug().Interface("config", d.config).Msg("running with config")
}

func (d *Hercules) Run() {
mux := http.NewServeMux()
prometheus.Unregister(collectors.NewGoCollector()) // Remove golang node defaults
mux.Handle("/metrics", middleware.MetricsMiddleware(d.conn, d.metricRegistry, promhttp.Handler()))
mux.Handle("/metrics", middleware.MetricsMiddleware(d.conn, d.metricRegistries, promhttp.Handler()))

srv := &http.Server{
Addr: ":" + d.config.Port,
Expand Down
6 changes: 3 additions & 3 deletions hercules-packages/example/nyc-taxi/1.0.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: nyc-taxi
name: nyc_taxi
version: 1.0

sources:
Expand All @@ -10,15 +10,15 @@ sources:

metrics:
gauge:
- name: nyc_taxi_pickup_location_fare_total
- name: pickup_location_fare_total
help: Total NYC fares for the month of July by pickup location
enabled: True
sql: select struct_pack(pickupLocation := PULocationID::text), sum(fare_amount) as val from nyc_yellow_taxi_june_2024 group by 1
labels:
- pickupLocation

summary:
- name: nyc_taxi_pickup_location_fares # Note this uses prometheus to do the histogram calculation. For better performance histograms can be pre-calculated and represented as a gauge.
- name: pickup_location_fares # Note this uses prometheus to do the histogram calculation. For better performance histograms can be pre-calculated and represented as a gauge.
help: Total NYC fares for the month of July by pickup location
enabled: True
sql: select struct_pack(pickupLocation := PULocationID::text), fare_amount as val from nyc_yellow_taxi_june_2024
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# - SNOWFLAKE.ACCOUNT_USAGE.DATA_TRANSFER_HISTORY
# - SNOWFLAKE.ACCOUNT_USAGE.DOCUMENT_AI_USAGE_HISTORY

name: snowflake-performance
name: snowflake
version: 1.0

macros:
Expand Down
5 changes: 4 additions & 1 deletion hercules.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ globalLabels:
- env: $ENV # Inject prometheus labels from env var

packages:
- package: hercules-packages/snowflake/performance/1.0.yml
- package: hercules-packages/snowflake/1.0.yml
variables:
yo: yee
metricPrefix: skt_
# - package: hercules-packages/example/nyc-taxi/1.0.yml
# - package: hercules-packages/example/tpch/1.0.yml
27 changes: 16 additions & 11 deletions pkg/herculesPackage/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/dbecorp/hercules/pkg/db"
"github.com/dbecorp/hercules/pkg/metrics"
"github.com/dbecorp/hercules/pkg/source"
herculestypes "github.com/dbecorp/hercules/pkg/types"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
)
Expand All @@ -16,39 +17,43 @@ type HerculesPackageVariables map[string]interface{}
// It can be downloaded from remote sources or shipped alongside hercules.

type Package struct {
Name string `json:"name"`
Version string `json:"version"`
Variables HerculesPackageVariables `json:"variables"`
Extensions db.Extensions `json:"extensions"`
Macros []db.Macro `json:"macros"`
Sources []source.Source `json:"sources"`
Metrics metrics.MetricDefinitions `json:"metrics"`
Name herculestypes.PackageName `json:"name"`
Version string `json:"version"`
Variables HerculesPackageVariables `json:"variables"`
MetricPrefix herculestypes.MetricPrefix `json:"metricPrefix"`
Extensions db.Extensions `json:"extensions"`
Macros []db.Macro `json:"macros"`
Sources []source.Source `json:"sources"`
Metrics metrics.MetricDefinitions `json:"metrics"`
// TODO -> Package-level secrets
}

func (p *Package) InitializeWithConnection(conn *sql.Conn) error {
if len(p.Name) > 0 {
log.Info().Interface("package", p.Name).Msg("initializing " + p.Name + " package")
log.Info().Interface("package", p.Name).Msg("initializing " + string(p.Name) + " package")
// Ensure extensions
db.EnsureExtensionsWithConnection(p.Extensions, conn)
// Ensure macros
db.EnsureMacrosWithConnection(p.Macros, conn)
// Ensure sources
source.InitializeSourcesWithConnection(p.Sources, conn)
log.Info().Interface("package", p.Name).Msg(p.Name + " package initialized")
log.Info().Interface("package", p.Name).Msg(string(p.Name) + " package initialized")
} else {
log.Trace().Msg("empty package detected - skipping initialization")
}
return nil
}

type PackageConfig struct {
Package string `json:"package"`
Variables HerculesPackageVariables `json:"variables"`
Package string `json:"package"`
Variables HerculesPackageVariables `json:"variables"`
MetricPrefix herculestypes.MetricPrefix `json:"metricPrefix"`
}

func (p *PackageConfig) GetPackage() (Package, error) {
pkg := &Package{}
pkg.Variables = p.Variables
pkg.MetricPrefix = p.MetricPrefix
// Try to get configuration from file
viper.SetConfigFile(p.Package)
viper.SetConfigType("yaml")
Expand Down
77 changes: 77 additions & 0 deletions pkg/metricRegistry/registry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package registry

import (
"database/sql"

"github.com/dbecorp/hercules/pkg/metrics"
herculestypes "github.com/dbecorp/hercules/pkg/types"
"github.com/rs/zerolog/log"
)

type MetricRegistry struct {
PackageName herculestypes.PackageName
MetricPrefix string
Gauge map[string]metrics.GaugeMetric
Counter map[string]metrics.CounterMetric
Summary map[string]metrics.SummaryMetric
Histogram map[string]metrics.HistogramMetric
}

func (mr *MetricRegistry) MaterializeWithConnection(conn *sql.Conn) error { // TODO -> Make this return a list of "materialization errors" if something fails
for _, gauge := range mr.Gauge {
err := gauge.MaterializeWithConnection(conn)
if err != nil {
log.Error().Err(err)
}
}

for _, histogram := range mr.Histogram {
err := histogram.MaterializeWithConnection(conn)
if err != nil {
log.Error().Err(err)
}
}

for _, summary := range mr.Summary {
err := summary.MaterializeWithConnection(conn)
if err != nil {
log.Error().Err(err)
}
}

for _, counter := range mr.Counter {
err := counter.MaterializeWithConnection(conn)
if err != nil {
log.Error().Err(err)
}
}
return nil
}

func NewMetricRegistry(definitions metrics.MetricDefinitions, meta herculestypes.MetricMetadata) *MetricRegistry {
r := MetricRegistry{}
r.PackageName = meta.PackageName
r.MetricPrefix = string(meta.MetricPrefix)
r.Gauge = make(map[string]metrics.GaugeMetric)
r.Histogram = make(map[string]metrics.HistogramMetric)
r.Summary = make(map[string]metrics.SummaryMetric)
r.Counter = make(map[string]metrics.CounterMetric)

for _, definition := range definitions.Gauge {
g := metrics.NewGaugeMetric(definition, meta)
r.Gauge[g.Definition.Name] = g
}
for _, definition := range definitions.Histogram {
h := metrics.NewHistogramMetric(definition, meta)
r.Histogram[h.Definition.Name] = h
}
for _, definition := range definitions.Summary {
s := metrics.NewSummaryMetric(definition, meta)
r.Summary[s.Definition.Name] = s
}
for _, definition := range definitions.Counter {
c := metrics.NewCounterMetric(definition, meta)
r.Counter[c.Definition.Name] = c
}
return &r
}
9 changes: 6 additions & 3 deletions pkg/metrics/counter.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"database/sql"

"github.com/dbecorp/hercules/pkg/labels"
herculestypes "github.com/dbecorp/hercules/pkg/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog/log"
)
Expand Down Expand Up @@ -41,7 +42,7 @@ func (m *CounterMetric) reregister() error {
return m.register()
}

func (m *CounterMetric) materializeWithConnection(conn *sql.Conn) error {
func (m *CounterMetric) MaterializeWithConnection(conn *sql.Conn) error {
m.reregister()
results, err := m.Definition.materializeWithConnection(conn)
if err != nil {
Expand All @@ -55,10 +56,12 @@ func (m *CounterMetric) materializeWithConnection(conn *sql.Conn) error {
return nil
}

func NewCounterMetric(definition CounterMetricDefinition, labels labels.GlobalLabels) CounterMetric {
func NewCounterMetric(definition CounterMetricDefinition, meta herculestypes.MetricMetadata) CounterMetric {
// TODO! Turn this into a generic function instead of copy/pasta
definition.Name = string(meta.MetricPrefix) + string(meta.PackageName) + "_" + definition.Name
metric := CounterMetric{
Definition: definition,
GlobalLabels: labels,
GlobalLabels: meta.Labels,
}
metric.register()
return metric
Expand Down
9 changes: 6 additions & 3 deletions pkg/metrics/gauge.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"database/sql"

"github.com/dbecorp/hercules/pkg/labels"
herculestypes "github.com/dbecorp/hercules/pkg/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog/log"
)
Expand Down Expand Up @@ -42,7 +43,7 @@ func (m *GaugeMetric) reregister() error {
return m.register()
}

func (m *GaugeMetric) materializeWithConnection(conn *sql.Conn) error {
func (m *GaugeMetric) MaterializeWithConnection(conn *sql.Conn) error {
m.reregister()
results, err := m.Definition.materializeWithConnection(conn)
if err != nil {
Expand All @@ -56,10 +57,12 @@ func (m *GaugeMetric) materializeWithConnection(conn *sql.Conn) error {
return nil
}

func NewGaugeMetric(definition GaugeMetricDefinition, labels labels.GlobalLabels) GaugeMetric {
func NewGaugeMetric(definition GaugeMetricDefinition, meta herculestypes.MetricMetadata) GaugeMetric {
// TODO! Turn this into a generic function instead of copy/pasta
definition.Name = string(meta.MetricPrefix) + string(meta.PackageName) + "_" + definition.Name
metric := GaugeMetric{
Definition: definition,
GlobalLabels: labels,
GlobalLabels: meta.Labels,
}
metric.register()
return metric
Expand Down
9 changes: 6 additions & 3 deletions pkg/metrics/histogram.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"database/sql"

"github.com/dbecorp/hercules/pkg/labels"
herculestypes "github.com/dbecorp/hercules/pkg/types"
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog/log"
)
Expand Down Expand Up @@ -43,7 +44,7 @@ func (m *HistogramMetric) reregister() error {
return m.register()
}

func (m *HistogramMetric) materializeWithConnection(conn *sql.Conn) error {
func (m *HistogramMetric) MaterializeWithConnection(conn *sql.Conn) error {
m.reregister()
results, err := m.Definition.materializeWithConnection(conn)
if err != nil {
Expand All @@ -57,10 +58,12 @@ func (m *HistogramMetric) materializeWithConnection(conn *sql.Conn) error {
return nil
}

func NewHistogramMetric(definition HistogramMetricDefinition, labels labels.GlobalLabels) HistogramMetric {
func NewHistogramMetric(definition HistogramMetricDefinition, meta herculestypes.MetricMetadata) HistogramMetric {
// TODO! Turn this into a generic function instead of copy/pasta
definition.Name = string(meta.MetricPrefix) + string(meta.PackageName) + "_" + definition.Name
metric := HistogramMetric{
Definition: definition,
GlobalLabels: labels,
GlobalLabels: meta.Labels,
}
metric.register()
return metric
Expand Down
Loading

0 comments on commit 6ef3ce7

Please sign in to comment.