diff --git a/README.md b/README.md index 9bb8982..e985632 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ -Hercules is a DuckDB-powered Prometheus exporter that supercharges metrics. +### Hercules is a Prometheus-compatible exporter that supercharges your metrics. -**Generate prometheus-compatible metrics** from parquet, csv files, json logs, data lakes, databases, http endpoints, and much more. +* **Generate prometheus-compatible metrics** from parquet, csv files, json logs, data lakes, databases, http endpoints, and much more. -**Generate enriched, labeled** metrics properly from the source; don't relabel using your favorite metrics database. +* **Generate enriched, labeled** metrics properly from the source; don't relabel using your favorite metrics database. -**Embrace** the pantheon of metrics harvesting using Prometheus-compatible scrape targets that easily tame [TPC-H benchmarks](https://www.tpc.org/information/benchmarks5.asp). +* **Embrace** the pantheon of metrics harvesting using Prometheus-compatible scrape targets that easily tame [TPC-H benchmarks](https://www.tpc.org/information/benchmarks5.asp). # Getting Started diff --git a/cmd/hercules/app.go b/cmd/hercules/app.go index 545d2bf..d6d74c1 100644 --- a/cmd/hercules/app.go +++ b/cmd/hercules/app.go @@ -13,8 +13,9 @@ import ( "github.com/dbecorp/hercules/pkg/config" "github.com/dbecorp/hercules/pkg/flock" herculespackage "github.com/dbecorp/hercules/pkg/herculesPackage" - metrics "github.com/dbecorp/hercules/pkg/metrics" + registry "github.com/dbecorp/hercules/pkg/metricRegistry" "github.com/dbecorp/hercules/pkg/middleware" + herculestypes "github.com/dbecorp/hercules/pkg/types" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -25,12 +26,12 @@ import ( var VERSION string type Hercules struct { - config config.Config - db *sql.DB - packages []herculespackage.Package - conn *sql.Conn - metricRegistry *metrics.MetricRegistry - debug bool + config config.Config + db *sql.DB + packages []herculespackage.Package + conn *sql.Conn + metricRegistries []*registry.MetricRegistry + debug bool } func (d *Hercules) configure() { @@ -80,13 +81,20 @@ func (d *Hercules) initializePackages() { } } -func (d *Hercules) initializeRegistry() { - // Merge metric definitions from all packages - metricDefinitions := metrics.MetricDefinitions{} +func (d *Hercules) initializeRegistries() { + // Register a registry for each package for _, pkg := range d.packages { - metricDefinitions.Merge(pkg.Metrics) + metricMetadata := herculestypes.MetricMetadata{ + PackageName: pkg.Name, + MetricPrefix: pkg.MetricPrefix, + Labels: d.config.InstanceLabels(), + } + if d.metricRegistries == nil { + d.metricRegistries = []*registry.MetricRegistry{registry.NewMetricRegistry(pkg.Metrics, metricMetadata)} + } else { + d.metricRegistries = append(d.metricRegistries, registry.NewMetricRegistry(pkg.Metrics, metricMetadata)) + } } - d.metricRegistry = metrics.NewMetricRegistry(metricDefinitions, d.config.InstanceLabels()) } func (d *Hercules) Initialize() { @@ -95,14 +103,14 @@ func (d *Hercules) Initialize() { d.initializeFlock() d.loadPackages() d.initializePackages() - d.initializeRegistry() + d.initializeRegistries() log.Debug().Interface("config", d.config).Msg("running with config") } func (d *Hercules) Run() { mux := http.NewServeMux() prometheus.Unregister(collectors.NewGoCollector()) // Remove golang node defaults - mux.Handle("/metrics", middleware.MetricsMiddleware(d.conn, d.metricRegistry, promhttp.Handler())) + mux.Handle("/metrics", middleware.MetricsMiddleware(d.conn, d.metricRegistries, promhttp.Handler())) srv := &http.Server{ Addr: ":" + d.config.Port, diff --git a/hercules-packages/example/nyc-taxi/1.0.yml b/hercules-packages/example/nyc-taxi/1.0.yml index 3282056..9e5f47d 100644 --- a/hercules-packages/example/nyc-taxi/1.0.yml +++ b/hercules-packages/example/nyc-taxi/1.0.yml @@ -1,4 +1,4 @@ -name: nyc-taxi +name: nyc_taxi version: 1.0 sources: @@ -10,7 +10,7 @@ sources: metrics: gauge: - - name: nyc_taxi_pickup_location_fare_total + - name: pickup_location_fare_total help: Total NYC fares for the month of July by pickup location enabled: True sql: select struct_pack(pickupLocation := PULocationID::text), sum(fare_amount) as val from nyc_yellow_taxi_june_2024 group by 1 @@ -18,7 +18,7 @@ metrics: - pickupLocation summary: - - name: nyc_taxi_pickup_location_fares # Note this uses prometheus to do the histogram calculation. For better performance histograms can be pre-calculated and represented as a gauge. + - name: pickup_location_fares # Note this uses prometheus to do the histogram calculation. For better performance histograms can be pre-calculated and represented as a gauge. help: Total NYC fares for the month of July by pickup location enabled: True sql: select struct_pack(pickupLocation := PULocationID::text), fare_amount as val from nyc_yellow_taxi_june_2024 diff --git a/hercules-packages/snowflake/performance/1.0.yml b/hercules-packages/snowflake/1.0.yml similarity index 99% rename from hercules-packages/snowflake/performance/1.0.yml rename to hercules-packages/snowflake/1.0.yml index 70120b5..fc48621 100644 --- a/hercules-packages/snowflake/performance/1.0.yml +++ b/hercules-packages/snowflake/1.0.yml @@ -12,7 +12,7 @@ # - SNOWFLAKE.ACCOUNT_USAGE.DATA_TRANSFER_HISTORY # - SNOWFLAKE.ACCOUNT_USAGE.DOCUMENT_AI_USAGE_HISTORY -name: snowflake-performance +name: snowflake version: 1.0 macros: diff --git a/hercules.yml b/hercules.yml index fbf34be..94491af 100644 --- a/hercules.yml +++ b/hercules.yml @@ -9,6 +9,9 @@ globalLabels: - env: $ENV # Inject prometheus labels from env var packages: - - package: hercules-packages/snowflake/performance/1.0.yml + - package: hercules-packages/snowflake/1.0.yml + variables: + yo: yee + metricPrefix: skt_ # - package: hercules-packages/example/nyc-taxi/1.0.yml # - package: hercules-packages/example/tpch/1.0.yml diff --git a/pkg/herculesPackage/package.go b/pkg/herculesPackage/package.go index 2474a04..63c2cfb 100644 --- a/pkg/herculesPackage/package.go +++ b/pkg/herculesPackage/package.go @@ -6,6 +6,7 @@ import ( "github.com/dbecorp/hercules/pkg/db" "github.com/dbecorp/hercules/pkg/metrics" "github.com/dbecorp/hercules/pkg/source" + herculestypes "github.com/dbecorp/hercules/pkg/types" "github.com/rs/zerolog/log" "github.com/spf13/viper" ) @@ -16,26 +17,27 @@ type HerculesPackageVariables map[string]interface{} // It can be downloaded from remote sources or shipped alongside hercules. type Package struct { - Name string `json:"name"` - Version string `json:"version"` - Variables HerculesPackageVariables `json:"variables"` - Extensions db.Extensions `json:"extensions"` - Macros []db.Macro `json:"macros"` - Sources []source.Source `json:"sources"` - Metrics metrics.MetricDefinitions `json:"metrics"` + Name herculestypes.PackageName `json:"name"` + Version string `json:"version"` + Variables HerculesPackageVariables `json:"variables"` + MetricPrefix herculestypes.MetricPrefix `json:"metricPrefix"` + Extensions db.Extensions `json:"extensions"` + Macros []db.Macro `json:"macros"` + Sources []source.Source `json:"sources"` + Metrics metrics.MetricDefinitions `json:"metrics"` // TODO -> Package-level secrets } func (p *Package) InitializeWithConnection(conn *sql.Conn) error { if len(p.Name) > 0 { - log.Info().Interface("package", p.Name).Msg("initializing " + p.Name + " package") + log.Info().Interface("package", p.Name).Msg("initializing " + string(p.Name) + " package") // Ensure extensions db.EnsureExtensionsWithConnection(p.Extensions, conn) // Ensure macros db.EnsureMacrosWithConnection(p.Macros, conn) // Ensure sources source.InitializeSourcesWithConnection(p.Sources, conn) - log.Info().Interface("package", p.Name).Msg(p.Name + " package initialized") + log.Info().Interface("package", p.Name).Msg(string(p.Name) + " package initialized") } else { log.Trace().Msg("empty package detected - skipping initialization") } @@ -43,12 +45,15 @@ func (p *Package) InitializeWithConnection(conn *sql.Conn) error { } type PackageConfig struct { - Package string `json:"package"` - Variables HerculesPackageVariables `json:"variables"` + Package string `json:"package"` + Variables HerculesPackageVariables `json:"variables"` + MetricPrefix herculestypes.MetricPrefix `json:"metricPrefix"` } func (p *PackageConfig) GetPackage() (Package, error) { pkg := &Package{} + pkg.Variables = p.Variables + pkg.MetricPrefix = p.MetricPrefix // Try to get configuration from file viper.SetConfigFile(p.Package) viper.SetConfigType("yaml") diff --git a/pkg/metricRegistry/registry.go b/pkg/metricRegistry/registry.go new file mode 100644 index 0000000..405abfe --- /dev/null +++ b/pkg/metricRegistry/registry.go @@ -0,0 +1,77 @@ +package registry + +import ( + "database/sql" + + "github.com/dbecorp/hercules/pkg/metrics" + herculestypes "github.com/dbecorp/hercules/pkg/types" + "github.com/rs/zerolog/log" +) + +type MetricRegistry struct { + PackageName herculestypes.PackageName + MetricPrefix string + Gauge map[string]metrics.GaugeMetric + Counter map[string]metrics.CounterMetric + Summary map[string]metrics.SummaryMetric + Histogram map[string]metrics.HistogramMetric +} + +func (mr *MetricRegistry) MaterializeWithConnection(conn *sql.Conn) error { // TODO -> Make this return a list of "materialization errors" if something fails + for _, gauge := range mr.Gauge { + err := gauge.MaterializeWithConnection(conn) + if err != nil { + log.Error().Err(err) + } + } + + for _, histogram := range mr.Histogram { + err := histogram.MaterializeWithConnection(conn) + if err != nil { + log.Error().Err(err) + } + } + + for _, summary := range mr.Summary { + err := summary.MaterializeWithConnection(conn) + if err != nil { + log.Error().Err(err) + } + } + + for _, counter := range mr.Counter { + err := counter.MaterializeWithConnection(conn) + if err != nil { + log.Error().Err(err) + } + } + return nil +} + +func NewMetricRegistry(definitions metrics.MetricDefinitions, meta herculestypes.MetricMetadata) *MetricRegistry { + r := MetricRegistry{} + r.PackageName = meta.PackageName + r.MetricPrefix = string(meta.MetricPrefix) + r.Gauge = make(map[string]metrics.GaugeMetric) + r.Histogram = make(map[string]metrics.HistogramMetric) + r.Summary = make(map[string]metrics.SummaryMetric) + r.Counter = make(map[string]metrics.CounterMetric) + + for _, definition := range definitions.Gauge { + g := metrics.NewGaugeMetric(definition, meta) + r.Gauge[g.Definition.Name] = g + } + for _, definition := range definitions.Histogram { + h := metrics.NewHistogramMetric(definition, meta) + r.Histogram[h.Definition.Name] = h + } + for _, definition := range definitions.Summary { + s := metrics.NewSummaryMetric(definition, meta) + r.Summary[s.Definition.Name] = s + } + for _, definition := range definitions.Counter { + c := metrics.NewCounterMetric(definition, meta) + r.Counter[c.Definition.Name] = c + } + return &r +} diff --git a/pkg/metrics/counter.go b/pkg/metrics/counter.go index cb6f57e..a440174 100644 --- a/pkg/metrics/counter.go +++ b/pkg/metrics/counter.go @@ -4,6 +4,7 @@ import ( "database/sql" "github.com/dbecorp/hercules/pkg/labels" + herculestypes "github.com/dbecorp/hercules/pkg/types" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog/log" ) @@ -41,7 +42,7 @@ func (m *CounterMetric) reregister() error { return m.register() } -func (m *CounterMetric) materializeWithConnection(conn *sql.Conn) error { +func (m *CounterMetric) MaterializeWithConnection(conn *sql.Conn) error { m.reregister() results, err := m.Definition.materializeWithConnection(conn) if err != nil { @@ -55,10 +56,12 @@ func (m *CounterMetric) materializeWithConnection(conn *sql.Conn) error { return nil } -func NewCounterMetric(definition CounterMetricDefinition, labels labels.GlobalLabels) CounterMetric { +func NewCounterMetric(definition CounterMetricDefinition, meta herculestypes.MetricMetadata) CounterMetric { + // TODO! Turn this into a generic function instead of copy/pasta + definition.Name = string(meta.MetricPrefix) + string(meta.PackageName) + "_" + definition.Name metric := CounterMetric{ Definition: definition, - GlobalLabels: labels, + GlobalLabels: meta.Labels, } metric.register() return metric diff --git a/pkg/metrics/gauge.go b/pkg/metrics/gauge.go index 8baae48..0d90237 100644 --- a/pkg/metrics/gauge.go +++ b/pkg/metrics/gauge.go @@ -4,6 +4,7 @@ import ( "database/sql" "github.com/dbecorp/hercules/pkg/labels" + herculestypes "github.com/dbecorp/hercules/pkg/types" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog/log" ) @@ -42,7 +43,7 @@ func (m *GaugeMetric) reregister() error { return m.register() } -func (m *GaugeMetric) materializeWithConnection(conn *sql.Conn) error { +func (m *GaugeMetric) MaterializeWithConnection(conn *sql.Conn) error { m.reregister() results, err := m.Definition.materializeWithConnection(conn) if err != nil { @@ -56,10 +57,12 @@ func (m *GaugeMetric) materializeWithConnection(conn *sql.Conn) error { return nil } -func NewGaugeMetric(definition GaugeMetricDefinition, labels labels.GlobalLabels) GaugeMetric { +func NewGaugeMetric(definition GaugeMetricDefinition, meta herculestypes.MetricMetadata) GaugeMetric { + // TODO! Turn this into a generic function instead of copy/pasta + definition.Name = string(meta.MetricPrefix) + string(meta.PackageName) + "_" + definition.Name metric := GaugeMetric{ Definition: definition, - GlobalLabels: labels, + GlobalLabels: meta.Labels, } metric.register() return metric diff --git a/pkg/metrics/histogram.go b/pkg/metrics/histogram.go index 1893a3f..7d40ec6 100644 --- a/pkg/metrics/histogram.go +++ b/pkg/metrics/histogram.go @@ -4,6 +4,7 @@ import ( "database/sql" "github.com/dbecorp/hercules/pkg/labels" + herculestypes "github.com/dbecorp/hercules/pkg/types" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog/log" ) @@ -43,7 +44,7 @@ func (m *HistogramMetric) reregister() error { return m.register() } -func (m *HistogramMetric) materializeWithConnection(conn *sql.Conn) error { +func (m *HistogramMetric) MaterializeWithConnection(conn *sql.Conn) error { m.reregister() results, err := m.Definition.materializeWithConnection(conn) if err != nil { @@ -57,10 +58,12 @@ func (m *HistogramMetric) materializeWithConnection(conn *sql.Conn) error { return nil } -func NewHistogramMetric(definition HistogramMetricDefinition, labels labels.GlobalLabels) HistogramMetric { +func NewHistogramMetric(definition HistogramMetricDefinition, meta herculestypes.MetricMetadata) HistogramMetric { + // TODO! Turn this into a generic function instead of copy/pasta + definition.Name = string(meta.MetricPrefix) + string(meta.PackageName) + "_" + definition.Name metric := HistogramMetric{ Definition: definition, - GlobalLabels: labels, + GlobalLabels: meta.Labels, } metric.register() return metric diff --git a/pkg/metrics/metric.go b/pkg/metrics/metric.go index 7461fdb..62b132b 100644 --- a/pkg/metrics/metric.go +++ b/pkg/metrics/metric.go @@ -4,7 +4,7 @@ import ( "database/sql" "github.com/dbecorp/hercules/pkg/db" - "github.com/dbecorp/hercules/pkg/labels" + herculestypes "github.com/dbecorp/hercules/pkg/types" "github.com/rs/zerolog/log" ) @@ -65,66 +65,6 @@ func (m *MetricDefinitions) Merge(metricDefinitions MetricDefinitions) { m.Histogram = append(m.Histogram, metricDefinitions.Histogram...) } -type MetricRegistry struct { - Gauge map[string]GaugeMetric - Counter map[string]CounterMetric - Summary map[string]SummaryMetric - Histogram map[string]HistogramMetric -} - -func (mr *MetricRegistry) MaterializeWithConnection(conn *sql.Conn) error { // TODO -> Make this return a list of "materialization errors" if something fails - for _, gauge := range mr.Gauge { - err := gauge.materializeWithConnection(conn) - if err != nil { - log.Error().Err(err) - } - } - - for _, histogram := range mr.Histogram { - err := histogram.materializeWithConnection(conn) - if err != nil { - log.Error().Err(err) - } - } - - for _, summary := range mr.Summary { - err := summary.materializeWithConnection(conn) - if err != nil { - log.Error().Err(err) - } - } - - for _, counter := range mr.Counter { - err := counter.materializeWithConnection(conn) - if err != nil { - log.Error().Err(err) - } - } - return nil -} - -func NewMetricRegistry(definitions MetricDefinitions, labels labels.GlobalLabels) *MetricRegistry { - r := MetricRegistry{} - r.Gauge = make(map[string]GaugeMetric) - r.Histogram = make(map[string]HistogramMetric) - r.Summary = make(map[string]SummaryMetric) - r.Counter = make(map[string]CounterMetric) - - for _, definition := range definitions.Gauge { - g := NewGaugeMetric(definition, labels) - r.Gauge[g.Definition.Name] = g - } - for _, definition := range definitions.Histogram { - h := NewHistogramMetric(definition, labels) - r.Histogram[h.Definition.Name] = h - } - for _, definition := range definitions.Summary { - s := NewSummaryMetric(definition, labels) - r.Summary[s.Definition.Name] = s - } - for _, definition := range definitions.Counter { - c := NewCounterMetric(definition, labels) - r.Counter[c.Definition.Name] = c - } - return &r +type MetricMetadata struct { + PackageName herculestypes.PackageName } diff --git a/pkg/metrics/summary.go b/pkg/metrics/summary.go index a1737e5..05b36ea 100644 --- a/pkg/metrics/summary.go +++ b/pkg/metrics/summary.go @@ -4,6 +4,7 @@ import ( "database/sql" "github.com/dbecorp/hercules/pkg/labels" + herculestypes "github.com/dbecorp/hercules/pkg/types" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog/log" ) @@ -47,7 +48,7 @@ func (m *SummaryMetric) reregister() error { return m.register() } -func (m *SummaryMetric) materializeWithConnection(conn *sql.Conn) error { +func (m *SummaryMetric) MaterializeWithConnection(conn *sql.Conn) error { m.reregister() results, err := m.Definition.materializeWithConnection(conn) if err != nil { @@ -61,10 +62,12 @@ func (m *SummaryMetric) materializeWithConnection(conn *sql.Conn) error { return nil } -func NewSummaryMetric(definition SummaryMetricDefinition, labels labels.GlobalLabels) SummaryMetric { +func NewSummaryMetric(definition SummaryMetricDefinition, meta herculestypes.MetricMetadata) SummaryMetric { + // TODO! Turn this into a generic function instead of copy/pasta + definition.Name = string(meta.MetricPrefix) + string(meta.PackageName) + "_" + definition.Name metric := SummaryMetric{ Definition: definition, - GlobalLabels: labels, + GlobalLabels: meta.Labels, } metric.register() return metric diff --git a/pkg/middleware/metric.go b/pkg/middleware/metric.go index 9256adc..216d417 100644 --- a/pkg/middleware/metric.go +++ b/pkg/middleware/metric.go @@ -4,12 +4,14 @@ import ( "database/sql" "net/http" - "github.com/dbecorp/hercules/pkg/metrics" + registry "github.com/dbecorp/hercules/pkg/metricRegistry" ) -func MetricsMiddleware(conn *sql.Conn, registry *metrics.MetricRegistry, next http.Handler) http.Handler { +func MetricsMiddleware(conn *sql.Conn, registries []*registry.MetricRegistry, next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - registry.MaterializeWithConnection(conn) + for _, registry := range registries { + registry.MaterializeWithConnection(conn) + } next.ServeHTTP(w, r) }) } diff --git a/pkg/types/metric.go b/pkg/types/metric.go new file mode 100644 index 0000000..87af6d8 --- /dev/null +++ b/pkg/types/metric.go @@ -0,0 +1,11 @@ +package herculestypes + +import "github.com/dbecorp/hercules/pkg/labels" + +type MetricPrefix string + +type MetricMetadata struct { + PackageName + MetricPrefix + Labels labels.GlobalLabels +} diff --git a/pkg/types/package.go b/pkg/types/package.go new file mode 100644 index 0000000..68a1219 --- /dev/null +++ b/pkg/types/package.go @@ -0,0 +1,4 @@ +package herculestypes + +// Packaging +type PackageName string diff --git a/todo/todo.md b/todo/todo.md index 9cdecb3..b6a207e 100644 --- a/todo/todo.md +++ b/todo/todo.md @@ -6,7 +6,7 @@ ## Database stuff - Support ATTACH-ing s3/gcs-based databases ❌ - - Support duckdb secrets registration ❌ + - Support duckdb-based secrets registration ❌ - Namespace all packages using a database so they don't collide ❌ ## Sources @@ -15,9 +15,15 @@ - Refresh on http-post (POST collector:9100/sources/$BLAH/refresh) ❌ - Support view-based sources ✅ +## Registries + - Make a registry for each package so they can be reloaded independently @ some point ✅ + - Reload registries via http ❌ + ## Metrics - Handle scalar values well ✅ - Continue materializing metrics if a single metric cannot be materialized instead of blowing up ✅ + - Prefix metric names with the package name ✅ + - Support custom prefixes via configuration ✅ - Genericize/interface metrics ❌ - Metric definition packages - Support named path groupings ❌ @@ -52,5 +58,4 @@ - CLI for authoring, linting, and publishing packages? Maybe? idk, not yet. ❌ ## Outstanding questions - - Should package names be prometheus metric name prefixes? - - + -