[Query] Add multi-zonal capability to remote reads

scripts/docker-integration-tests/query_fanout/docker-compose.yml

@@ -44,5 +44,27 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml"
+  dbnode-cluster-c:
+    expose:
+      - "9000-9004"
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:29000-29004:9000-9004"
+      - "0.0.0.0:22379-22380:2379-2380"
+    networks:
+      - backend
+    image: "m3dbnode_integration:${REVISION}"
+  coordinator-cluster-c:
+    expose:
+      - "7201"
+      - "7203"
+    ports:
+      - "0.0.0.0:27201:7201"
+      - "0.0.0.0:27203:7203"
+    networks:
+      - backend
+    image: "m3coordinator_integration:${REVISION}"
+    volumes:
+      - "./m3coordinator-cluster-c.yml:/etc/m3coordinator/m3coordinator.yml"
 networks:
   backend:

scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-a.yml

@@ -16,7 +16,11 @@ metrics:
 rpc:
   enabled: true
   listenAddress: "0.0.0.0:7202"
-  remoteListenAddresses: ["coordinator-cluster-b:7202"]
+  remotes:
+    - name: "cluster-b"
+      remoteListenAddresses: ["coordinator-cluster-b:7202"]
+    - name: "cluster-c"
+      remoteListenAddresses: ["coordinator-cluster-c:7202"]
 
 clusters:
   - namespaces:

scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-b.yml

@@ -16,7 +16,11 @@ metrics:
 rpc:
   enabled: true
   listenAddress: "0.0.0.0:7202"
-  remoteListenAddresses: ["coordinator-cluster-a:7202"]
+  remotes:
+    - name: "cluster-a"
+      remoteListenAddresses: ["coordinator-cluster-a:7202"]
+    - name: "cluster-c"
+      remoteListenAddresses: ["coordinator-cluster-c:7202"]
 
 clusters:
   - namespaces:

scripts/docker-integration-tests/query_fanout/m3coordinator-cluster-c.yml

@@ -0,0 +1,49 @@
+listenAddress:
+  type: "config"
+  value: "0.0.0.0:7201"
+
+metrics:
+  scope:
+    prefix: "coordinator"
+  prometheus:
+    handlerPath: /metrics
+    listenAddress: 0.0.0.0:7203 # until https://github.com/m3db/m3/issues/682 is resolved
+  sanitization: prometheus
+  samplingRate: 1.0
+  extended: none
+
+# Fanout queries to remote clusters
+rpc:
+  enabled: true
+  listenAddress: "0.0.0.0:7202"
+  remotes:
+    - name: "cluster-a"
+      remoteListenAddresses: ["coordinator-cluster-a:7202"]
+    - name: "cluster-b"
+      remoteListenAddresses: ["coordinator-cluster-b:7202"]
+
+clusters:
+  - namespaces:
+      - namespace: agg
+        type: aggregated
+        retention: 10h
+        resolution: 15s
+      - namespace: unagg
+        type: unaggregated
+        retention: 10m
+    client:
+      config:
+        service:
+          env: default_env
+          zone: embedded
+          service: m3db
+          cacheDir: /var/lib/m3kv
+          etcdClusters:
+            - zone: embedded
+              endpoints:
+                - dbnode-cluster-c:2379
+      writeConsistencyLevel: majority
+      readConsistencyLevel: unstrict_majority
+
+tagOptions:
+  idScheme: quoted

scripts/docker-integration-tests/query_fanout/test.sh

@@ -14,6 +14,9 @@ docker-compose -f ${COMPOSE_FILE} up -d coordinator-cluster-a
 docker-compose -f ${COMPOSE_FILE} up -d dbnode-cluster-b
 docker-compose -f ${COMPOSE_FILE} up -d coordinator-cluster-b
 
+docker-compose -f ${COMPOSE_FILE} up -d dbnode-cluster-c
+docker-compose -f ${COMPOSE_FILE} up -d coordinator-cluster-c
+
 # think of this as a defer func() in golang
 function defer {
   docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes
@@ -26,6 +29,9 @@ DBNODE_HOST=dbnode-cluster-a DBDNODE_PORT=9000 DBNODE_HEALTH_PORT=9002 COORDINAT
 DBNODE_HOST=dbnode-cluster-b DBDNODE_PORT=19000 DBNODE_HEALTH_PORT=19002 COORDINATOR_PORT=17201 \
  setup_single_m3db_node
 
+DBNODE_HOST=dbnode-cluster-c DBDNODE_PORT=29000 DBNODE_HEALTH_PORT=29002 COORDINATOR_PORT=27201 \
+ setup_single_m3db_node
+
 echo "Write data to cluster a"
 curl -vvvsS -X POST 0.0.0.0:9003/writetagged -d '{
   "namespace": "unagg",
@@ -74,18 +80,42 @@ curl -vvvsS -X POST 0.0.0.0:19003/writetagged -d '{
   }
 }'
 
+echo "Write data to cluster c"
+curl -vvvsS -X POST 0.0.0.0:29003/writetagged -d '{
+  "namespace": "unagg",
+  "id": "{__name__=\"test_metric\",cluster=\"cluster-c\",endpoint=\"/request\"}",
+  "tags": [
+    {
+      "name": "__name__",
+      "value": "test_metric"
+    },
+    {
+      "name": "cluster",
+      "value": "cluster-c"
+    },
+    {
+      "name": "endpoint",
+      "value": "/request"
+    }
+  ],
+  "datapoint": {
+    "timestamp":'"$(date +"%s")"',
+    "value": 42.123456789
+  }
+}'
+
 function read {
   RESPONSE=$(curl "http://0.0.0.0:7201/api/v1/query?query=test_metric")
   ACTUAL=$(echo $RESPONSE | jq .data.result[].metric.cluster)
-  test "$(echo $ACTUAL)" = '"cluster-a" "cluster-b"'
+  test "$(echo $ACTUAL)" = '"cluster-a" "cluster-b" "cluster-c"'
 }
 
 ATTEMPTS=5 TIMEOUT=1 retry_with_backoff read
 
 function read_sum {
   RESPONSE=$(curl "http://0.0.0.0:7201/api/v1/query?query=sum(test_metric)")
   ACTUAL=$(echo $RESPONSE | jq .data.result[].value[1])
-  test $ACTUAL = '"84.246913578"'
+  test $ACTUAL = '"126.370370367"'
 }
 
 ATTEMPTS=5 TIMEOUT=1 retry_with_backoff read_sum
@@ -146,10 +176,38 @@ curl -vvvsS -X POST 0.0.0.0:19003/writetagged -d '{
   }
 }'
 
+echo "Write remote tagged data to cluster c"
+curl -vvvsS -X POST 0.0.0.0:29003/writetagged -d '{
+  "namespace": "unagg",
+  "id": "{__name__=\"test_metric\",cluster=\"cluster-c\",endpoint=\"/request\",third-cluster=\"third\"}",
+  "tags": [
+    {
+      "name": "__name__",
+      "value": "test_metric"
+    },
+    {
+      "name": "cluster",
+      "value": "cluster-c"
+    },
+    {
+      "name": "endpoint",
+      "value": "/request"
+    },
+    {
+      "name": "third-cluster",
+      "value": "third"
+    }
+  ],
+  "datapoint": {
+    "timestamp":'"$(date +"%s")"',
+    "value": 42.123456789
+  }
+}'
+
 function complete_tags {
   RESPONSE=$(curl "http://0.0.0.0:7201/api/v1/labels")
   ACTUAL=$(echo $RESPONSE | jq .data[])
-  test "$(echo $ACTUAL)" = '"__name__" "cluster" "endpoint" "local-only" "remote-only"'
+  test "$(echo $ACTUAL)" = '"__name__" "cluster" "endpoint" "local-only" "remote-only" "third-cluster"'
 }
 
 ATTEMPTS=5 TIMEOUT=1 retry_with_backoff complete_tags

src/cmd/services/m3query/config/config.go

@@ -406,6 +406,18 @@ type ClusterManagementConfiguration struct {
 	Etcd etcdclient.Configuration `yaml:"etcd"`
 }
 
+// Remotes is a set of remote host configurations.
+type Remotes []Remote
+
+// Remote is the configuration for a single remote host.
+type Remote struct {
+	// Name is the name for the remote zone.
+	Name string `yaml:"name"`
+	// RemoteListenAddresses is the remote listen addresses to call for remote
+	// coordinator calls in the remote zone.
+	RemoteListenAddresses []string `yaml:"remoteListenAddresses"`
+}
+
 // RPCConfiguration is the RPC configuration for the coordinator for
 // the GRPC server used for remote coordinator to coordinator calls.
 type RPCConfiguration struct {
@@ -415,8 +427,14 @@ type RPCConfiguration struct {
 	// ListenAddress is the RPC server listen address.
 	ListenAddress string `yaml:"listenAddress"`
 
-	// RemoteListenAddresses is the remote listen addresses to call for remote
-	// coordinator calls.
+	// Remotes are the configuration settings for remote coordinator zones.
+	Remotes Remotes `yaml:"remotes"`
+
+	// RemoteListenAddresses is the remote listen addresses to call for
+	// remote coordinator calls.
+	//
+	// NB: this is deprecated in favor of using RemoteZones, as setting
+	// RemoteListenAddresses will only allow for a single remote zone to be used.
 	RemoteListenAddresses []string `yaml:"remoteListenAddresses"`
 }