Merge pull request #351 from airbrake/tdigests

route_sender: replace min/max with TDigests

airbrake-ruby.gemspec

@@ -27,6 +27,8 @@ DESC
 
   s.required_ruby_version = '>= 2.0'
 
+  s.add_dependency 'tdigest', '= 0.1.1'
+
   s.add_development_dependency 'rspec', '~> 3'
   s.add_development_dependency 'rake', '~> 10'
   s.add_development_dependency 'pry', '~> 0'

lib/airbrake-ruby/response.rb

@@ -23,7 +23,7 @@ def self.parse(response, logger)
 
       begin
         case code
-        when 200
+        when 200, 204
           logger.debug("#{LOG_LABEL} #{name} (#{code}): #{body}")
           { response.msg => response.body }
         when 201

lib/airbrake-ruby/route_sender.rb

@@ -1,20 +1,76 @@
+require 'tdigest'
+require 'base64'
+
 module Airbrake
   # RouteSender aggregates information about requests and periodically sends
   # collected data to Airbrake.
   # @since v2.13.0
   class RouteSender
+    # Monkey-patch https://github.com/castle/tdigest to pack with Big Endian
+    # (instead of Little Endian) since our backend wants it.
+    #
+    # @see https://github.com/castle/tdigest/blob/master/lib/tdigest/tdigest.rb
+    # @since v2.13.0
+    # @api private
+    module TDigestBigEndianness
+      refine TDigest::TDigest do
+        # rubocop:disable Metrics/AbcSize
+        def as_small_bytes
+          size = @centroids.size
+          output = [self.class::SMALL_ENCODING, compression, size]
+          x = 0
+          # delta encoding allows saving 4-bytes floats
+          mean_arr = @centroids.map do |_, c|
+            val = c.mean - x
+            x = c.mean
+            val
+          end
+          output += mean_arr
+          # Variable length encoding of numbers
+          c_arr = @centroids.each_with_object([]) do |(_, c), arr|
+            k = 0
+            n = c.n
+            while n < 0 || n > 0x7f
+              b = 0x80 | (0x7f & n)
+              arr << b
+              n = n >> 7
+              k += 1
+              raise 'Unreasonable large number' if k > 6
+            end
+            arr << n
+          end
+          output += c_arr
+          output.pack("NGNg#{size}C#{size}")
+        end
+        # rubocop:enable Metrics/AbcSize
+      end
+    end
+
+    using TDigestBigEndianness
+
     # The key that represents a route.
     RouteKey = Struct.new(:method, :route, :statusCode, :time)
 
     # RouteStat holds data that describes a route's performance.
-    RouteStat = Struct.new(:count, :sum, :sumsq, :min, :max) do
+    RouteStat = Struct.new(:count, :sum, :sumsq, :tdigest) do
       # @param [Integer] count The number of requests
       # @param [Float] sum The sum of request duration in milliseconds
       # @param [Float] sumsq The squared sum of request duration in milliseconds
-      # @param [Float] min The minimal request duration
-      # @param [Float] max The maximum request duration
-      def initialize(count: 0, sum: 0.0, sumsq: 0.0, min: 0.0, max: 0.0)
-        super(count, sum, sumsq, min, max)
+      # @param [TDigest::TDigest] tdigest
+      def initialize(count: 0, sum: 0.0, sumsq: 0.0, tdigest: TDigest::TDigest.new)
+        super(count, sum, sumsq, tdigest)
+      end
+
+      # @return [Hash{String=>Object}] the route stat as a hash with compressed
+      #   and serialized as binary base64 tdigest
+      def to_h
+        tdigest.compress!
+        {
+          'count' => count,
+          'sum' => sum,
+          'sumsq' => sumsq,
+          'tDigest' => Base64.strict_encode64(tdigest.as_small_bytes)
+        }
       end
     end
 
@@ -66,8 +122,7 @@ def increment_stats(stat, dur)
       stat.sum += ms
       stat.sumsq += ms * ms
 
-      stat.min = ms if ms < stat.min || stat.min == 0
-      stat.max = ms if ms > stat.max
+      stat.tdigest.push(ms)
     end
 
     def schedule_flush(promise)

spec/response_spec.rb

@@ -5,7 +5,7 @@
     let(:out) { StringIO.new }
     let(:logger) { Logger.new(out) }
 
-    [200, 201].each do |code|
+    [200, 201, 204].each do |code|
       context "when response code is #{code}" do
         it "logs response body" do
           described_class.parse(OpenStruct.new(code: code, body: '{}'), logger)

spec/route_sender_spec.rb

@@ -48,10 +48,10 @@
             {"routes":\[
               {"method":"GET","route":"/foo","statusCode":200,
                "time":"2018-01-01T00:00:00\+00:00","count":1,"sum":24.0,
-               "sumsq":576.0,"min":24.0,"max":24.0},
+               "sumsq":576.0,"tDigest":"AAAAAkBZAAAAAAAAAAAAAUHAAAAB"},
               {"method":"GET","route":"/foo","statusCode":200,
                "time":"2018-01-01T00:01:00\+00:00","count":1,"sum":10.0,
-               "sumsq":100.0,"min":10.0,"max":10.0}\]}
+               "sumsq":100.0,"tDigest":"AAAAAkBZAAAAAAAAAAAAAUEgAAAB"}\]}
           \z|x
         )
       ).to have_been_made
@@ -67,10 +67,10 @@
             {"routes":\[
               {"method":"GET","route":"/foo","statusCode":200,
                "time":"2018-01-01T00:00:00\+00:00","count":1,"sum":24.0,
-               "sumsq":576.0,"min":24.0,"max":24.0},
+               "sumsq":576.0,"tDigest":"AAAAAkBZAAAAAAAAAAAAAUHAAAAB"},
               {"method":"POST","route":"/foo","statusCode":200,
                "time":"2018-01-01T00:00:00\+00:00","count":1,"sum":10.0,
-               "sumsq":100.0,"min":10.0,"max":10.0}\]}
+               "sumsq":100.0,"tDigest":"AAAAAkBZAAAAAAAAAAAAAUEgAAAB"}\]}
           \z|x
         )
       ).to have_been_made