Skip to content

Commit

Permalink
[feed] improve robustness and reduce timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
@xeroc
xeroc committed Jun 30, 2016
1 parent 1e887f1 commit 4943b97
Showing 1 changed file with 30 additions and 93 deletions.
123 changes: 30 additions & 93 deletions scripts/pricefeeds/feedsources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ class FeedSource() :
def __init__(self, scaleVolumeBy=1.0,
enable=True,
allowFailure=False,
timeout=20,
timeout=5,
quotes=[],
bases=[],
**kwargs):
Expand All @@ -39,8 +39,12 @@ def fetch(self):
self.updateCache(feed)
return feed
except Exception as e:
print("\nWe encountered an error loading live data. Trying to recover from cache {1}! ({0})".format(str(e), type(self).__name__))
return self.recoverFromCache()
print("\n{1} We encountered an error loading live data. Trying to recover from cache! ({0})".format(str(e), type(self).__name__))
try:
return self.recoverFromCache()
except:
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")

def today(self):
return datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d")
Expand All @@ -64,7 +68,7 @@ def getCacheFileName(self):

def updateCache(self, feed):
with open(self.getCacheFileName(), 'w') as fp:
state = json.dump(feed, fp)
json.dump(feed, fp)


class BitcoinIndonesia(FeedSource) :
Expand All @@ -88,10 +92,7 @@ def _fetch(self):
"volume" : float(result["vol_" + quote.lower()]) * self.scaleVolumeBy}
feed[base]["response"] = response.json()
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -127,12 +128,7 @@ def _fetch(self):
feed[base][quote] = {"price" : float(result["response"]["entity"]["last_price"]),
"volume" : float(result["response"]["entity"]["vol"]) * self.scaleVolumeBy}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
import traceback
traceback.print_exc()
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -161,10 +157,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(result[marketName]["ticker"]["last"])),
"volume" : (float(result[marketName]["ticker"]["vol"]) * self.scaleVolumeBy)}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -196,10 +189,7 @@ def _fetch(self):
print("\nFetched data from {0} is empty!".format(type(self).__name__))
continue
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand All @@ -225,10 +215,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(result[quote.lower() + "_" + base.lower()]["last"])),
"volume" : (float(result[quote.lower() + "_" + base.lower()]["vol_" + base.lower()]) * self.scaleVolumeBy)}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand All @@ -255,10 +242,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(result[marketName]["last"])),
"volume" : (float(result[marketName]["quoteVolume"]) * self.scaleVolumeBy)}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand All @@ -285,10 +269,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(thisMarket["Last"])),
"volume" : (float(thisMarket["Volume"]) * self.scaleVolumeBy)}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -325,10 +306,7 @@ def _fetch(self):
# feed[ config.core_symbol ][ self.bts_yahoo_map(a) ] = { "price" : (1/float(yahooprices[i])),
# "volume" : 1.0, }
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand All @@ -353,10 +331,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(result["last"])),
"volume" : (float(result["total_vol"]))}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand All @@ -381,10 +356,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(result["ticker"]["last"])),
"volume" : (float(result["ticker"]["vol"]) * self.scaleVolumeBy)}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand All @@ -409,10 +381,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(result["ticker"]["last"])),
"volume" : (float(result["ticker"]["vol"]) * self.scaleVolumeBy)}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -442,10 +411,7 @@ def _fetch(self):
feed[base][quote] = {"price" : (float(result["ticker"]["last"])),
"volume" : (float(result["ticker"]["vol"]) * self.scaleVolumeBy)}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -477,10 +443,7 @@ def _fetch(self):
feed[base][quote] = {"price" : result["rates"][quote],
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -512,10 +475,7 @@ def _fetch(self):
feed[base][quote] = {"price" : result["quotes"][base + quote],
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand All @@ -537,10 +497,7 @@ def _fetch(self):
feed[base][quote] = {"price" : result["rates"][quote],
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -569,10 +526,7 @@ def _fetch(self):
feed[base][quote] = {"price" : result[base][quote],
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -609,11 +563,7 @@ def _fetch(self):
feed[base]['ALTCAP.X'] = {"price" : btc_altcapx_price,
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
print(e)
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -648,11 +598,7 @@ def _fetch(self):
feed[base]['ALTCAP.X'] = {"price" : btc_altcapx_price,
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
print(e)
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -701,10 +647,7 @@ def _fetch(self):
"volume" : (float(ticker[market]["quoteVolume"]) * self.scaleVolumeBy)}
feed[base]["response"] = ticker[market]
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -741,10 +684,7 @@ def _fetch(self):
feed[base][quote] = {"price" : sum(prices) / len(prices),
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed


Expand Down Expand Up @@ -774,7 +714,7 @@ def _fetch(self):
)
response = requests.get(url=url, headers=_request_headers, timeout=self.timeout)
data = response.json()
if not "dataset" in data:
if "dataset" not in data:
raise Exception("Feed has not returned a dataset for url: %s" % url)
d = data["dataset"]
if len(d["data"]):
Expand All @@ -783,8 +723,5 @@ def _fetch(self):
feed[base][quote] = {"price" : sum(prices) / len(prices),
"volume" : 1.0}
except Exception as e:
print("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
if not self.allowFailure:
sys.exit("\nExiting due to exchange importance!")
return
raise Exception("\nError fetching results from {1}! ({0})".format(str(e), type(self).__name__))
return feed

0 comments on commit 4943b97

Please sign in to comment.