Optimizations for api_explorer_get_top_proxies (#51)

api/explorer.py

@@ -1,6 +1,8 @@
 import itertools
 import datetime
 import json
+from multiprocessing import Pool
+from functools import partial
 import connexion.problem
 from services.bitshares_websocket_client import client as bitshares_ws_client
 from services.bitshares_elasticsearch_client import client as bitshares_es_client
@@ -385,7 +387,8 @@ def get_market_chart_data(base, quote):
 
     return data
 
-def get_top_proxies():
+@cache.memoize()
+def _get_all_proxies():
     holders = _get_holders()
 
     total_votes = reduce(lambda acc, h: acc + int(h['balance']), holders, 0)
@@ -407,12 +410,23 @@ def get_top_proxies():
 
     return proxies
 
+def get_top_proxies(start=0, limit=10):
+    proxies = _get_all_proxies()
+    return proxies[start:start+limit]
+
+def get_proxy_count():
+    proxies = _get_all_proxies()
+    return len(proxies)
+
 def _get_accounts_by_chunks_via_es(account_ids, chunk_size=1000):
-    all_accounts = []
-    for i in xrange(0, len(account_ids), chunk_size):
-        accounts = bitshares_es_client.get_accounts(account_ids[i:i+chunk_size], size=chunk_size)
-        all_accounts.extend(accounts)
-    return all_accounts
+    f = partial(_get_accounts_chunck_via_es, account_ids, chunk_size)
+    pool = Pool()
+    accounts = pool.map(f, xrange(0, len(account_ids), chunk_size))
+    flattened_accounts = list(itertools.chain.from_iterable(accounts))
+    return flattened_accounts
+
+def _get_accounts_chunck_via_es(account_ids, chunk_size, i):
+    return bitshares_es_client.get_accounts(account_ids[i:i+chunk_size], size=chunk_size)
 
 def _get_accounts_by_chunks_via_ws(account_ids, chunk_size=1000):
     all_accounts = []
@@ -488,7 +502,6 @@ def _get_formatted_proxy_votes(proxies, vote_id):
 
 def get_witnesses_votes():
     proxies = get_top_proxies()
-    proxies = proxies[:10]
     proxies = bitshares_ws_client.request('database', 'get_objects', [[ p['id'] for p in proxies ]])
 
     witnesses = get_witnesses()
@@ -512,7 +525,6 @@ def get_witnesses_votes():
 
 def get_workers_votes():
     proxies = get_top_proxies()
-    proxies = proxies[:10]
     proxies = bitshares_ws_client.request('database', 'get_objects', [[ p['id'] for p in proxies ]])
 
     workers = get_workers()
@@ -538,7 +550,6 @@ def get_workers_votes():
 
 def get_committee_votes():
     proxies = get_top_proxies()
-    proxies = proxies[:10]
     proxies = bitshares_ws_client.request('database', 'get_objects', [[ p['id'] for p in proxies ]])
 
     committee_members = get_committee_members()

swagger/paths_explorer.yaml

@@ -522,6 +522,19 @@ paths:
     get:
       description: Get the top proxies
       operationId: api.explorer.get_top_proxies
+      parameters:
+        - in: query
+          name: start
+          default: 0
+          type: integer
+          required: false
+          description: Pagination start
+        - in: query
+          name: limit
+          default: 10
+          type: integer
+          required: false
+          description: Number of records to retreive
       responses:
         '200':
           description: Top X proxies
@@ -531,6 +544,19 @@ paths:
         - api
         - proxy
         - governance
+  "/proxy_count":
+    get:
+      description: Get the number of proxies
+      operationId: api.explorer.get_proxy_count
+      responses:
+        '200':
+          description: Number of proxies
+        '500':
+          description: Error processing parameters
+      tags:
+        - api
+        - proxy
+        - governance
   "/top_holders":
     get:
       description: Get the top holders

tests/test_api_explorer.tavern.yaml

@@ -555,6 +555,20 @@ stages:
               followers: !anyint
               bts_weight_percentage: !anyfloat
 ---
+test_name: Retrieve /proxy_count
+stages:
+  - name: Make sure we have the right content
+    request:
+      url: "{host}/proxy_count"
+      method: GET
+    response:
+      status_code: 200
+      body:
+        $ext:
+          function: tests.util:validate_unique_value
+          extra_kwargs:
+            unique_value: !anyint
+---
 test_name: Retrieve /top_holders
 stages:
   - name: Make sure we have the right content