From b65557df089c55fc4f8a5e0d691ec39134e87f34 Mon Sep 17 00:00:00 2001
From: Zapata <marco.tessari@gmail.com>
Date: Sat, 3 Aug 2019 21:43:00 +0200
Subject: [PATCH] Optimizations for api_explorer_get_top_proxies (#51)

  - add start and limit parameters to /top_proxy
 - add /proxy_count
 - parallelize ES accounts retrieval
---
 api/explorer.py                     | 29 ++++++++++++++++++++---------
 swagger/paths_explorer.yaml         | 26 ++++++++++++++++++++++++++
 tests/test_api_explorer.tavern.yaml | 14 ++++++++++++++
 3 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/api/explorer.py b/api/explorer.py
index 558f524..73aee63 100644
--- a/api/explorer.py
+++ b/api/explorer.py
@@ -1,6 +1,8 @@
 import itertools
 import datetime
 import json
+from multiprocessing import Pool
+from functools import partial
 from services.bitshares_websocket_client import client as bitshares_ws_client
 from services.bitshares_elasticsearch_client import client as bitshares_es_client
 from services.cache import cache
@@ -381,7 +383,8 @@ def get_market_chart_data(base, quote):
 
     return data
 
-def get_top_proxies():
+@cache.memoize()
+def _get_all_proxies():
     holders = _get_holders()
     
     total_votes = reduce(lambda acc, h: acc + int(h['balance']), holders, 0)
@@ -403,12 +406,23 @@ def get_top_proxies():
 
     return proxies
 
+def get_top_proxies(start=0, limit=10):
+    proxies = _get_all_proxies()
+    return proxies[start:start+limit]
+
+def get_proxy_count():
+    proxies = _get_all_proxies()
+    return len(proxies)
+
 def _get_accounts_by_chunks_via_es(account_ids, chunk_size=1000):
-    all_accounts = []
-    for i in xrange(0, len(account_ids), chunk_size):
-        accounts = bitshares_es_client.get_accounts(account_ids[i:i+chunk_size], size=chunk_size)
-        all_accounts.extend(accounts)
-    return all_accounts
+    f = partial(_get_accounts_chunck_via_es, account_ids, chunk_size)
+    pool = Pool()
+    accounts = pool.map(f, xrange(0, len(account_ids), chunk_size))
+    flattened_accounts = list(itertools.chain.from_iterable(accounts))
+    return flattened_accounts
+
+def _get_accounts_chunck_via_es(account_ids, chunk_size, i):
+    return bitshares_es_client.get_accounts(account_ids[i:i+chunk_size], size=chunk_size)
 
 def _get_accounts_by_chunks_via_ws(account_ids, chunk_size=1000):
     all_accounts = []
@@ -484,7 +498,6 @@ def _get_formatted_proxy_votes(proxies, vote_id):
 
 def get_witnesses_votes():
     proxies = get_top_proxies()
-    proxies = proxies[:10]
     proxies = bitshares_ws_client.request('database', 'get_objects', [[ p['id'] for p in proxies ]])
 
     witnesses = get_witnesses()
@@ -508,7 +521,6 @@ def get_witnesses_votes():
 
 def get_workers_votes():
     proxies = get_top_proxies()
-    proxies = proxies[:10]
     proxies = bitshares_ws_client.request('database', 'get_objects', [[ p['id'] for p in proxies ]])
 
     workers = get_workers()
@@ -534,7 +546,6 @@ def get_workers_votes():
 
 def get_committee_votes():
     proxies = get_top_proxies()
-    proxies = proxies[:10]
     proxies = bitshares_ws_client.request('database', 'get_objects', [[ p['id'] for p in proxies ]])
 
     committee_members = get_committee_members()
diff --git a/swagger/paths_explorer.yaml b/swagger/paths_explorer.yaml
index 0481f66..bbefe44 100644
--- a/swagger/paths_explorer.yaml
+++ b/swagger/paths_explorer.yaml
@@ -462,6 +462,19 @@ paths:
     get:
       description: Get the top proxies
       operationId: api.explorer.get_top_proxies
+      parameters:
+        - in: query
+          name: start
+          default: 0
+          type: integer
+          required: false
+          description: Pagination start
+        - in: query
+          name: limit
+          default: 10
+          type: integer
+          required: false
+          description: Number of records to retreive
       responses:
         '200':
           description: Top X proxies
@@ -471,6 +484,19 @@ paths:
         - api
         - proxy
         - governance
+  "/proxy_count":
+    get:
+      description: Get the number of proxies
+      operationId: api.explorer.get_proxy_count
+      responses:
+        '200':
+          description: Number of proxies
+        '500':
+          description: Error processing parameters
+      tags:
+        - api
+        - proxy
+        - governance
   "/top_holders":
     get:
       description: Get the top holders
diff --git a/tests/test_api_explorer.tavern.yaml b/tests/test_api_explorer.tavern.yaml
index c4f8b19..e6431db 100644
--- a/tests/test_api_explorer.tavern.yaml
+++ b/tests/test_api_explorer.tavern.yaml
@@ -523,6 +523,20 @@ stages:
               followers: !anyint
               bts_weight_percentage: !anyfloat
 ---
+test_name: Retrieve /proxy_count
+stages:
+  - name: Make sure we have the right content
+    request:
+      url: "{host}/proxy_count"
+      method: GET
+    response:
+      status_code: 200
+      body:
+        $ext:
+          function: tests.util:validate_unique_value
+          extra_kwargs:
+            unique_value: !anyint
+---
 test_name: Retrieve /top_holders
 stages:
   - name: Make sure we have the right content