Merge pull request #37 from dbecorp/more-packaging

More packaging

cmd/hercules/app.go

@@ -30,13 +30,15 @@ type Hercules struct {
 	packages       []herculespackage.Package
 	conn           *sql.Conn
 	metricRegistry *metrics.MetricRegistry
+	debug          bool
 }
 
 func (d *Hercules) configure() {
 	log.Debug().Msg("configuring Hercules")
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
 	debug := os.Getenv(config.DEBUG)
 	if debug != "" && (debug == "true" || debug == "1" || debug == "True") {
+		d.debug = true
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	}
 	trace := os.Getenv(config.TRACE)
@@ -122,7 +124,9 @@ func (d *Hercules) Run() {
 	if err := srv.Shutdown(ctx); err != nil {
 		log.Fatal().Stack().Err(err).Msg("server forced to shutdown")
 		d.db.Close()
-		os.Remove(d.config.Db)
+		if !d.debug {
+			os.Remove(d.config.Db)
+		}
 	}
 	d.db.Close()
 	os.Remove(d.config.Db)

hercules-packages/example/nyc-taxi/1.0.yml

@@ -1,5 +1,5 @@
 name: nyc-taxi
-version: 1.0.0
+version: 1.0
 
 sources:
   - name: nyc_yellow_taxi_june_2024
@@ -10,9 +10,24 @@ sources:
 
 metrics:
   gauge:
-    - name: nyc_pickup_location_fare_total
-      help: Total NYC fares for the month of August by pickup location
+    - name: nyc_taxi_pickup_location_fare_total
+      help: Total NYC fares for the month of July by pickup location
       enabled: True
       sql: select struct_pack(pickupLocation := PULocationID::text), sum(fare_amount) as val from nyc_yellow_taxi_june_2024 group by 1
       labels:
         - pickupLocation
+
+  summary:
+    - name: nyc_taxi_pickup_location_fares # Note this uses prometheus to do the histogram calculation. For better performance histograms can be pre-calculated and represented as a gauge.
+      help: Total NYC fares for the month of July by pickup location
+      enabled: True
+      sql: select struct_pack(pickupLocation := PULocationID::text), fare_amount as val from nyc_yellow_taxi_june_2024
+      labels:
+        - pickupLocation
+      objectives:
+        - 0.001
+        - 0.05
+        - 0.01
+        - 0.5
+        - 0.9
+        - 0.99

hercules-packages/example/tpch/1.0.yml

@@ -1,2 +1,2 @@
 name: tpch
-version: 1.0.0
+version: 1.0

hercules-packages/example/us-zips/1.0.yml

@@ -0,0 +1,17 @@
+name: us-zips
+version: 1.0
+
+sources:
+  - name: united_states_zip_codes
+    type: csv
+    source: https://raw.githubusercontent.com/scpike/us-state-county-zip/refs/heads/master/geo-data.csv
+    materialize: true
+    refreshIntervalSeconds: 100
+
+metrics:
+  gauge:
+    - name: united_states_zip_codes_total
+      help: The number of US zip codes
+      sql: select struct_pack(state := state) as labels, count(*) as value from united_states_zip_codes group by 1;
+      labels:
+        - state

hercules.yml

@@ -9,7 +9,7 @@ globalLabels:
   - env: $ENV # Inject prometheus labels from env var
 
 packages:
-  - package: hercules-packages/snowflake/performance/1.0.yml
+  # - package: hercules-packages/snowflake/performance/1.0.yml
   - package: hercules-packages/example/nyc-taxi/1.0.yml
 
 extensions:
@@ -26,14 +26,6 @@ extensions:
 
 macros:
   - sql: create or replace macro one() AS (SELECT 1);
-  - sql: create or replace macro two() AS (SELECT 2);
-
-sources:
-  - name: united_states_zip_codes
-    type: csv
-    source: https://raw.githubusercontent.com/scpike/us-state-county-zip/refs/heads/master/geo-data.csv
-    materialize: true
-    refreshIntervalSeconds: 100
 
 metrics:
   gauge:
@@ -43,16 +35,3 @@ metrics:
       sql: select struct_pack(user := 'yo') as labels, one();
       labels:
         - user
-
-    - name: two_two_two_two
-      help: Testing the two() macro
-      enabled: true
-      sql: select struct_pack(user := 'yeeee') as labels, two();
-      labels:
-        - user
-
-    - name: united_states_zip_codes_total
-      help: The number of US zip codes
-      sql: select struct_pack(state := state) as labels, count(*) as value from united_states_zip_codes group by 1;
-      labels:
-        - state

todo/todo.md

@@ -50,3 +50,7 @@
 ## Package distribution stuff
     - Pull from public/s3-backed, nicely-named package registry ❌
     - CLI for authoring, linting, and publishing packages? Maybe? idk, not yet. ❌
+
+## Outstanding questions
+    - Should package names be prometheus metric name prefixes?
+    -