m3-query: scalar operator on timestamp yielding wrong result

[stevehorsfield]

Service: m3-query

Context

I'm trying to build a Grafana single-stat panel that shows the number of seconds since the last successful Prometheus configuration reload.

Problem statement

The following query works against Prometheus:

time() - prometheus_config_last_reload_success_timestamp_seconds

At the time of writing, the result coming back is around 900 seconds.

Aganst M3, the result is negative 63.7 billion!

Response data

{
  "xhrStatus": "complete",
  "request": {
    "method": "GET",
    "url": "api/datasources/proxy/6/api/v1/query?query=time()%20-%20prometheus_config_last_reload_success_timestamp_seconds&time=1565795507"
  },
  "response": {
    "status": "success",
    "data": {
      "resultType": "vector",
      "result": [
        {
          "metric": {
            "application": "prometheus",
            "ec2_availability_zone": "us-east-1a",
            "instance": "REDACTED",
            "job": "prometheus:ec2-discovery"
          },
          "value": [
            1565795507,
            "-63701391334.34311"
          ],
          "values": [
            [
              1565795507,
              "-63701391334.34311"
            ]
          ]
        },
        {
          "metric": {
            "application": "prometheus",
            "ec2_availability_zone": "us-east-1b",
            "instance": "REDACTED",
            "job": "prometheus:ec2-discovery"
          },
          "value": [
            1565795507,
            "-63701391324.18791"
          ],
          "values": [
            [
              1565795507,
              "-63701391324.18791"
            ]
          ]
        },
        {
          "metric": {
            "application": "prometheus",
            "ec2_availability_zone": "us-east-1c",
            "instance": "REDACTED",
            "job": "prometheus:ec2-discovery"
          },
          "value": [
            1565795507,
            "-63701391318.88583"
          ],
          "values": [
            [
              1565795507,
              "-63701391318.88583"
            ]
          ]
        }
      ]
    }
  }
}

Verification

Executing this query directly against Prometheus yields the expected outcome.

M3 Configuration

Placement

{
    "num_shards": 32,
    "replication_factor": 3,
    "instances": [
        {
            "id": "m3-data0.domain",
            "isolation_group": "us-east-1a",
            "zone": "embedded",
            "weight": 100,
            "endpoint": "m3-data0.domain:9000",
            "hostname": "m3-data0.domain",
            "port": 9000
        },
        {
            "id": "m3-data1.domain",
            "isolation_group": "us-east-1b",
            "zone": "embedded",
            "weight": 100,
            "endpoint": "m3-data1.domain:9000",
            "hostname": "m3-data1.domain",
            "port": 9000
        },
        {
            "id": "m3-data2.domain",
            "isolation_group": "us-east-1c",
            "zone": "embedded",
            "weight": 100,
            "endpoint": "m3-data2.domain:9000",
            "hostname": "m3-data2.domain",
            "port": 9000
        }
    ]
}

Namespaces

{
  "name": "default",
  "options": {
    "bootstrapEnabled": true,
    "flushEnabled": true,
    "writesToCommitLog": true,
    "cleanupEnabled": true,
    "snapshotEnabled": true,
    "repairEnabled": false,
    "retentionOptions": {
      "retentionPeriodDuration": "792h",
      "blockSizeDuration": "3h",
      "bufferFutureDuration": "10m",
      "bufferPastDuration": "10m",
      "blockDataExpiry": true,
      "blockDataExpiryAfterNotAccessPeriodDuration": "5m"
    },
    "indexOptions": {
      "enabled": true,
      "blockSizeDuration": "6h"
    }
  }
}
{
    "namespaceName": "agg_100days_1h",
    "retentionTime": "2400h"
}
{
    "namespaceName": "agg_1year_4h",
    "retentionTime": "17520h"
}