diff --git a/docs/howto-exchanges-detailed.rst b/docs/howto-exchanges-detailed.rst index c42ba703..b55f6dec 100644 --- a/docs/howto-exchanges-detailed.rst +++ b/docs/howto-exchanges-detailed.rst @@ -31,7 +31,7 @@ still support exchanges that require more confirmations for deposits. We provide a so called *delayed* full node which accepts two additional parameters for the configuration besides those already available with the -standard daemon (read :doc:`full`). +standard daemon. * `trusted-node` RPC endpoint of a trusted validating node (required) * `delay-block-count` Number of blocks to delay before advancing chain state (required) diff --git a/docs/howto-monitor-operations.rst b/docs/howto-monitor-operations.rst new file mode 100644 index 00000000..b8db4ad6 --- /dev/null +++ b/docs/howto-monitor-operations.rst @@ -0,0 +1,31 @@ +*************************************************** +Howto Monitor the blockchain for certain operations +*************************************************** + +Operations in blocks can be monitored relatively easy by using the +`block_stream` (for entire blocks) for `stream` (for specific +operations) generators. + +The following example will only show ``transfer`` operations on the +blockchain: + +.. code-block:: python + + from grapheneapi.grapheneclient import GrapheneClient + from pprint import pprint + + class Config(): + witness_url = "ws://testnet.bitshares.eu/ws" + + if __name__ == '__main__': + client = GrapheneClient(Config) + for b in client.ws.stream("transfer"): + pprint(b) + +Note that you can define a starting block and instead of waiting for +sufficient confirmations (irreversible blocks), you can also consider +the real *head* block with: + +.. code-block:: python + + for b in client.ws.stream("transfer", start=199924, mode="head"): diff --git a/docs/index.rst b/docs/index.rst index 37300fe6..a2146a9a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,6 +43,7 @@ Tutorials .. toctree:: :maxdepth: 1 + howto-monitor-operations howto-exchanges howto-exchanges-detailed diff --git a/examples/approve_proposal.py b/examples/approve_proposal.py new file mode 100644 index 00000000..c9ed8bb7 --- /dev/null +++ b/examples/approve_proposal.py @@ -0,0 +1,46 @@ +from grapheneapi.grapheneclient import GrapheneClient +from pprint import pprint + + +class config(): + witness_url = "ws://testnet.bitshares.eu/ws" + wallet_host = "localhost" + wallet_port = 8092 + + +if __name__ == '__main__': + client = GrapheneClient(config) + graphene = client.rpc + + # Get current fees + core_asset = graphene.get_asset("1.3.0") + committee_account = graphene.get_account("committee-account") + proposals = client.ws.get_proposed_transactions(committee_account["id"]) + + for proposal in proposals: + print("Proposal: %s" % proposal["id"]) + + prop_op = proposal["proposed_transaction"]["operations"] + + if len(prop_op) > 1: + print(" - [Warning] This proposal has more than 1 operation") + + if graphene._confirm("Approve?"): + tx = graphene.approve_proposal( + "xeroc", + proposal["id"], + {"active_approvals_to_add": + ["committee-member-1", + "committee-member-2", + "committee-member-3", + "committee-member-4", + "committee-member-5", + "committee-member-6", + "committee-member-7", + "init0", + "init1", + "init2", + "init3", + ]}, + True) + pprint(tx) diff --git a/examples/change_fee.py b/examples/change_fee.py new file mode 100644 index 00000000..778e7cc3 --- /dev/null +++ b/examples/change_fee.py @@ -0,0 +1,56 @@ +from grapheneapi.grapheneclient import GrapheneClient +from graphenebase.transactions import getOperationNameForId +from pprint import pprint +from deepdiff import DeepDiff + +proposer = "xeroc" +expiration = "2016-05-17T09:00:00" +price_per_kbyte = 0 +everythin_flat_fee = 0.001 +broadcast = True + + +class Wallet(): + wallet_host = "localhost" + wallet_port = 8092 + +if __name__ == '__main__': + graphene = GrapheneClient(Wallet) + obj = graphene.getObject("2.0.0") + current_fees = obj["parameters"]["current_fees"]["parameters"] + old_fees = obj["parameters"]["current_fees"] + scale = obj["parameters"]["current_fees"]["scale"] / 1e4 + + # General change of parameter + changes = {} + for f in current_fees: + if ("price_per_kbyte" in f[1] and f[1]["price_per_kbyte"] != 0): + print("Changing operation %s[%d]" % (getOperationNameForId( + f[0]), f[0])) + changes[getOperationNameForId(f[0])] = f[1].copy() + changes[getOperationNameForId(f[0])]["price_per_kbyte"] = int( + price_per_kbyte / scale * 1e5) + if ("fee" in f[1] and f[1]["fee"] != 0): + print("Changing operation %s[%d]" % (getOperationNameForId( + f[0]), f[0])) + changes[getOperationNameForId(f[0])] = f[1].copy() + changes[getOperationNameForId(f[0])]["fee"] = int( + everythin_flat_fee / scale * 1e5) + + # overwrite / set specific fees + changes["transfer"]["price_per_kbyte"] = int(0) + # changes["account_update"]["price_per_kbyte"] = int( 5 / scale * 1e5) + + print("=" * 80) + tx = graphene.rpc.propose_fee_change(proposer, + expiration, + changes, + broadcast) + proposed_ops = tx["operations"][0][1]["proposed_ops"][0] + new_fees = proposed_ops["op"][1]["new_parameters"]["current_fees"] + + pprint(DeepDiff(old_fees, new_fees)) + + if not broadcast: + print("=" * 80) + print("Set broadcast to 'True' if the transaction shall be broadcast!") diff --git a/examples/transfer_back_to_issuer.py b/examples/transfer_back_to_issuer.py new file mode 100644 index 00000000..6b71daf5 --- /dev/null +++ b/examples/transfer_back_to_issuer.py @@ -0,0 +1,59 @@ +from grapheneapi import GrapheneClient +from grapheneexchange import GrapheneExchange +from graphenebase import transactions +from pprint import pprint + +issuer = "xeroc" +from_account = "maker" +to_account = "xeroc" +asset = "LIVE" +amount = 100.0 +wifs = [""] +witness_url = "ws://testnet.bitshares.eu/ws" + + +def constructSignedTransaction(ops): + ops = transactions.addRequiredFees(client.ws, ops, "1.3.0") + ref_block_num, ref_block_prefix = transactions.getBlockParams(client.ws) + expiration = transactions.formatTimeFromNow(30) + tx = transactions.Signed_Transaction( + ref_block_num=ref_block_num, + ref_block_prefix=ref_block_prefix, + expiration=expiration, + operations=ops + ) + w = tx.sign(wifs, chain=client.getChainInfo()) + return w + + +#: Connetion Settings +class Config(): + witness_url = witness_url + + +if __name__ == '__main__': + config = Config + client = GrapheneClient(config) + + issuer = client.ws.get_account(issuer) + from_account = client.ws.get_account(from_account) + to_account = client.ws.get_account(to_account) + asset = client.ws.get_asset(asset) + amount = int(amount * 10 ** asset["precision"]) + + ops = [] + op = transactions.Override_transfer(**{ + "fee": {"amount": 0, + "asset_id": "1.3.0"}, + "issuer": issuer["id"], + "from": from_account["id"], + "to": to_account["id"], + "amount": {"amount": amount, + "asset_id": asset["id"]}, + "extensions": [] + }) + ops.append(transactions.Operation(op)) + + tx = constructSignedTransaction(ops) + pprint(transactions.JsonObj(tx)) + print(client.ws.broadcast_transaction(transactions.JsonObj(tx), api="network_broadcast")) diff --git a/grapheneapi/grapheneapi.py b/grapheneapi/grapheneapi.py index 58277f8d..24021c5f 100644 --- a/grapheneapi/grapheneapi.py +++ b/grapheneapi/grapheneapi.py @@ -1,5 +1,7 @@ import sys import json +import logging +log = logging.getLogger(__name__) try: import requests @@ -113,8 +115,7 @@ def rpcexec(self, payload): rtype: json raises RPCConnection: if no connction can be made raises UnauthorizedError: if the user is not authorized - raise ValueError: if the API returns a non-JSON formated - answer + raise ValueError: if the API returns a non-JSON formated answer It is not recommended to use this method directly, unless you know what you are doing. All calls available to the API diff --git a/grapheneapi/grapheneclient.py b/grapheneapi/grapheneclient.py index 85c9330d..2c4f75c8 100644 --- a/grapheneapi/grapheneclient.py +++ b/grapheneapi/grapheneclient.py @@ -2,35 +2,8 @@ from .graphenews import GrapheneWebsocket from collections import OrderedDict -import logging as log - -#: max number of objects to chache -max_cache_objects = 50 - - -class LimitedSizeDict(OrderedDict): - """ This class limits the size of the objectMap - """ - - def __init__(self, *args, **kwds): - self.size_limit = kwds.pop("size_limit", max_cache_objects) - OrderedDict.__init__(self, *args, **kwds) - self._check_size_limit() - - def __setitem__(self, key, value): - OrderedDict.__setitem__(self, key, value) - self._check_size_limit() - - def _check_size_limit(self): - if self.size_limit is not None: - while len(self) > self.size_limit: - self.popitem(last=False) - - def __getitem__(self, key): - """ keep the element longer in the memory by moving it to the end - """ - self.move_to_end(key) - return OrderedDict.__getitem__(self, key) +import logging +log = logging.getLogger(__name__) class ExampleConfig() : @@ -105,6 +78,10 @@ class Config(GrapheneWebsocketProtocol): ## Note the dependency #: ``onAccountUpdate()`` to be called watch_accounts = ["fabian", "nathan"] + #: Assets you want to watch. Changes will be used to call + #: ``onAssetUpdate()``. + watch_assets = ["USD"] + #: Markets to watch. Changes to these will result in the method #: ``onMarketUpdate()`` to be called watch_markets = ["USD:CORE"] @@ -184,6 +161,20 @@ def onAccountUpdate(self, data): """ pass + def onAssetUpdate(self, data): + """ This method is called when any of the assets in watch_assets + changes. The changes of the following objects are monitored: + + * Asset object (``1.3.x``) + * Dynamic Asset data (``2.3.x``) + * Bitasset data (``2.4.x``) + + Hence, this method needs to distinguish these three + objects! + + """ + pass + def onMarketUpdate(self, data): """ This method will be called if a subscribed market sees an event (registered to through ``watch_markets``). @@ -414,6 +405,10 @@ def __init__(self, config): except: raise Exception("Couldn't load assets for market %s" % market) + if not quote or not base: + raise Exception("Couldn't load assets for market %s" + % market) + if "id" in quote and "id" in base: if "onMarketUpdate" in available_features: self.markets.update({ @@ -432,6 +427,19 @@ def __init__(self, config): log.warn("Market assets could not be found: %s" % market) self.setMarketCallBack(self.markets) + + if ("watch_assets" in available_features): + assets = [] + for asset in config.watch_assets: + a = self.ws.get_asset(asset) + if not a: + log.warning("The asset %s does not exist!" % a) + + if ("onAssetUpdate" in available_features): + a["callback"] = config.onAssetUpdate + assets.append(a) + self.setAssetDispatcher(assets) + if "onRegisterHistory" in available_features: self.setEventCallbacks( {"registered-history": config.onRegisterHistory}) @@ -490,6 +498,30 @@ def getChainInfo(self): "core_symbol" : core_asset["symbol"], "chain_id" : chain_id} + def getObject(self, oid): + """ Get an Object either from Websocket store (if available) or + from RPC connection. + """ + if self.ws : + [_instance, _type, _id] = oid.split(".") + if (not (oid in self.ws.objectMap) or + _instance == "1" and _type == "7"): # force refresh orders + data = self.ws.get_object(oid) + self.ws.objectMap[oid] = data + else: + data = self.ws.objectMap[oid] + if len(data) == 1 : + return data[0] + else: + return data + else : + return self.rpc.get_object(oid)[0] + + def get_object(self, oid): + """ Identical to ``getObject`` + """ + return self.getObject(oid) + """ Forward these calls to Websocket API """ def setEventCallbacks(self, callbacks): @@ -513,6 +545,13 @@ def setMarketCallBack(self, markets): """ self.ws.setMarketCallBack(markets) + def setAssetDispatcher(self, markets): + """ Internally used to register Market update callbacks + """ + self.ws.setAssetDispatcher(markets) + + """ Connect to Websocket and run asynchronously + """ def connect(self): """ Only *connect* to the websocket server. Does **not** run the subsystem. @@ -529,22 +568,3 @@ def run(self): """ Connect to Websocket server **and** run the subsystem """ self.connect() self.run_forever() - - def getObject(self, oid): - """ Get an Object either from Websocket store (if available) or - from RPC connection. - """ - if self.ws : - [_instance, _type, _id] = oid.split(".") - if (not (oid in self.ws.objectMap) or - _instance == "1" and _type == "7"): # force refresh orders - data = self.ws.get_object(oid) - self.ws.objectMap[oid] = data - else: - data = self.ws.objectMap[oid] - if len(data) == 1 : - return data[0] - else: - return data - else : - return self.rpc.get_object(oid)[0] diff --git a/grapheneapi/graphenews.py b/grapheneapi/graphenews.py index a53de652..9e370f4d 100644 --- a/grapheneapi/graphenews.py +++ b/grapheneapi/graphenews.py @@ -16,27 +16,44 @@ from .graphenewsprotocol import GrapheneWebsocketProtocol from .graphenewsrpc import GrapheneWebsocketRPC +import logging +log = logging.getLogger(__name__) + #: max number of objects to chache -max_cache_objects = 50 class LimitedSizeDict(OrderedDict): - """ This class limits the size of the objectMap + """ This class limits the size of the objectMap to + ``max_cache_objects`` (default_ 50). + + All objects received are stored in the objectMap and get_object + calls will lookup most objects from this structure """ + max_cache_objects = 50 + def __init__(self, *args, **kwds): - self.size_limit = kwds.pop("size_limit", max_cache_objects) + if "max_cache_objects" in kwds: + self.max_cache_objects = kwds["max_cache_objects"] + self.size_limit = kwds.pop("size_limit", self.max_cache_objects) OrderedDict.__init__(self, *args, **kwds) self._check_size_limit() def __setitem__(self, key, value): OrderedDict.__setitem__(self, key, value) + self.move_to_end(key, last=False) self._check_size_limit() def _check_size_limit(self): if self.size_limit is not None: while len(self) > self.size_limit: - self.popitem(last=False) + self.popitem(last=False) # False -> FIFO + +# def __getitem__(self, key): +# """ keep the element longer in the memory by moving it to the end +# """ +# # self.move_to_end(key, last=False) +# return OrderedDict.__getitem__(self, key) class GrapheneWebsocket(GrapheneWebsocketRPC): @@ -51,13 +68,25 @@ class GrapheneWebsocket(GrapheneWebsocketRPC): `autobahn.asyncio.websocket`. """ - def __init__(self, url, username, password, + def __init__(self, url, username="", password="", proto=GrapheneWebsocketProtocol): + """ Open A GrapheneWebsocket connection that can handle + notifications though asynchronous calls. + + :param str url: Url to the websocket server + :param str username: Username for login + :param str password: Password for login + :param GrapheneWebsocketProtocol proto: (optional) Protocol that inherits ``GrapheneWebsocketProtocol`` + """ ssl, host, port, resource, path, params = parseWsUrl(url) - GrapheneWebsocketRPC.__init__(self, url, username, password) self.url = url self.username = username self.password = password + + # Open another RPC connection to execute calls + GrapheneWebsocketRPC.__init__(self, url, username, password) + + # Parameters for another connection for asynchronous notifications self.ssl = ssl self.host = host self.port = port @@ -68,6 +97,74 @@ def __init__(self, url, username, password, self.proto.objectMap = self.objectMap # this is a reference self.factory = None + def get_object(self, oid): + """ Get_Object as a passthrough from get_objects([array]) + Attention: This call requires GrapheneAPI because it is a non-blocking + JSON query + + :param str oid: Object ID to fetch + """ + return self.get_objects([oid])[0] + + def getObject(self, oid): + """ Lookup objects from the object storage and if not available, + request object from the API + """ + if self.objectMap is not None and oid in self.objectMap: + return self.objectMap[oid] + else: + data = self.get_object(oid) + self.objectMap[oid] = data + return data + + def connect(self) : + """ Create websocket factory by Autobahn + """ + self.factory = WebSocketClientFactory(self.url) + self.factory.protocol = self.proto + + def run_forever(self) : + """ Run websocket forever and wait for events. + + This method will try to keep the connection alive and try an + autoreconnect if the connection closes. + """ + if not issubclass(self.factory.protocol, GrapheneWebsocketProtocol) : + raise Exception("When using run(), we need websocket " + + "notifications which requires the " + + "configuration/protocol to inherit " + + "'GrapheneWebsocketProtocol'") + + loop = asyncio.get_event_loop() + # forward loop into protocol so that we can issue a reset from the + # protocol: + self.factory.protocol.setLoop(self.factory.protocol, loop) + + while True : + try : + if self.ssl : + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + coro = loop.create_connection(self.factory, self.host, + self.port, ssl=context) + else : + coro = loop.create_connection(self.factory, self.host, + self.port, ssl=self.ssl) + + loop.run_until_complete(coro) + loop.run_forever() + except KeyboardInterrupt: + break + except: + pass + + log.error("Trying to re-connect in 10 seconds!") + time.sleep(10) + + log.info("Good bye!") + loop.close() + def setObjectCallbacks(self, callbacks) : """ Define Callbacks on Objects for websocket connections @@ -125,70 +222,32 @@ def setMarketCallBack(self, markets) : .. code-block:: python - setMarketCallBack(["USD:EUR", "GOLD:USD"]) + market = {"quote" : quote["id"], + "base" : base["id"], + "base_symbol" : base["symbol"], + "quote_symbol" : quote["symbol"], + "callback": print} + setMarketCallBack([market]) """ self.proto.markets = markets - def get_object(self, oid): - """ Get_Object as a passthrough from get_objects([array]) - Attention: This call requires GrapheneAPI because it is a non-blocking - JSON query + def setAssetDispatcher(self, assets) : + """ Define Callbacks on Asset Events for websocket connections - :param str oid: Object ID to fetch - """ - return self.get_objects([oid])[0] + :param markets: Array of Assets to register to + :type markets: array of asset pairs - def getObject(self, oid): - if self.objectMap is not None and oid in self.objectMap: - return self.objectMap[oid] - else: - data = self.get_object(oid) - self.objectMap[oid] = data - return data + Example - def connect(self) : - """ Create websocket factory by Autobahn - """ - self.factory = WebSocketClientFactory(self.url) - self.factory.protocol = self.proto + .. code-block:: python - def run_forever(self) : - """ Run websocket forever and wait for events. + asset = {"id" : "1.3.121", + "bitasset_data_id": "2.4.21", + "dynamic_asset_data_id": "2.3.121", + "symbol" : "USD", + "callback": print} + setAssetCallBack([asset]) - This method will try to keep the connection alive and try an - autoreconnect if the connection closes. """ - if not issubclass(self.factory.protocol, GrapheneWebsocketProtocol) : - raise Exception("When using run(), we need websocket " + - "notifications which requires the " + - "configuration/protocol to inherit " + - "'GrapheneWebsocketProtocol'") - - loop = asyncio.get_event_loop() - # forward loop into protocol so that we can issue a reset from the - # protocol: - self.factory.protocol.setLoop(self.factory.protocol, loop) - - while True : - try : - if self.ssl : - context = ssl.create_default_context() - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - coro = loop.create_connection(self.factory, self.host, - self.port, ssl=context) - else : - coro = loop.create_connection(self.factory, self.host, - self.port, ssl=self.ssl) - - loop.run_until_complete(coro) - loop.run_forever() - except KeyboardInterrupt: - break - - print("Trying to re-connect in 10 seconds!") - time.sleep(10) - - print("Good bye!") - loop.close() + self.proto.assets = assets diff --git a/grapheneapi/graphenewsprotocol.py b/grapheneapi/graphenewsprotocol.py index f4f43a8f..621bb557 100644 --- a/grapheneapi/graphenewsprotocol.py +++ b/grapheneapi/graphenewsprotocol.py @@ -1,5 +1,9 @@ import json from functools import partial +import warnings +import logging +log = logging.getLogger(__name__) + try: from autobahn.asyncio.websocket import WebSocketClientProtocol @@ -30,6 +34,9 @@ class GrapheneWebsocketProtocol(WebSocketClientProtocol): #: Markets to subscribe to markets = [] + #: Assets to subscribe to + assets = [] + #: Storage of Objects to reduce latency and load objectMap = None @@ -47,6 +54,12 @@ class GrapheneWebsocketProtocol(WebSocketClientProtocol): def __init__(self): pass + def _get_request_id(self): + self.request_id += 1 + return self.request_id + + """ Basic RPC connection + """ def wsexec(self, params, callback=None): """ Internally used method to execute calls @@ -55,26 +68,15 @@ def wsexec(self, params, callback=None): of the answer (defaults to ``None``) """ request = {"request" : {}, "callback" : None} - self.request_id += 1 - request["id"] = self.request_id + request["id"] = self._get_request_id() request["request"]["id"] = self.request_id request["request"]["method"] = "call" request["request"]["params"] = params request["callback"] = callback self.requests.update({self.request_id: request}) -# print(json.dumps(request["request"],indent=4)) -# print(request["request"]) + log.debug(request["request"]) self.sendMessage(json.dumps(request["request"]).encode('utf8')) - def eventcallback(self, name): - """ Call an event callback - - :param str name: Name of the event - """ - if (name in self.onEventCallbacks and - callable(self.onEventCallbacks[name])): - self.onEventCallbacks[name](self) - def register_api(self, name): """ Register to an API of graphene @@ -105,6 +107,8 @@ def _login(self): """ self.wsexec([1, "login", [self.username, self.password]]) + """ Subscriptions + """ def subscribe_to_accounts(self, account_ids, *args): """ Subscribe to account ids @@ -119,9 +123,8 @@ def subscribe_to_markets(self, dummy=None): """ for m in self.markets: market = self.markets[m] - self.request_id += 1 self.wsexec([0, "subscribe_to_market", - [self.request_id, + [self._get_request_id(), market["quote"], market["base"]]]) @@ -130,6 +133,7 @@ def subscribe_to_objects(self, *args): * ``self.database_callbacks`` * ``self.accounts`` + * ``self.assets`` and set the subscription callback. """ @@ -139,44 +143,70 @@ def subscribe_to_objects(self, *args): self.database_callbacks_ids.update({ handle: self.database_callbacks[handle]}) + asset_ids = set() + for m in self.assets: + asset_ids.add(m["id"]) + if "bitasset_data_id" in m: + asset_ids.add(m["bitasset_data_id"]) + if "dynamic_asset_data_id" in m: + asset_ids.add(m["dynamic_asset_data_id"]) + handles.append(partial(self.getObjectscb, list(asset_ids), None)) + if self.accounts: handles.append(partial(self.subscribe_to_accounts, self.accounts)) - self.request_id += 1 self.wsexec([self.api_ids["database"], "set_subscribe_callback", - [self.request_id, False]], handles) + [self._get_request_id(), False]], handles) - def getAccountHistory(self, account_id, callback, - start="1.11.0", stop="1.11.0", limit=100): - """ Get Account history History and call callback + """ Objects + """ + def getObjectcb(self, oid, callback, *args): + """ Get an Object from the internal object storage if available + or otherwise retrieve it from the API. - :param account-id account_id: Account ID to read the history for - :param fnt callback: Callback to execute with the response - :param historyID start: Start of the history (defaults to ``1.11.0``) - :param historyID stop: Stop of the history (defaults to ``1.11.0``) - :param historyID stop: Limit entries by (defaults to ``100``, max ``100``) - :raises ValueError: if the account id is incorrectly formatted + :param object-id oid: Object ID to retrieve + :param fnt callback: Callback to call if object has been received """ - if account_id[0:4] == "1.2." : - self.wsexec([self.api_ids["history"], - "get_account_history", - [account_id, start, 100, stop]], - callback) - else : - raise ValueError("getAccountHistory expects an account" + - "id of the form '1.2.x'!") + self.getObjectscb([oid], callback, *args) - def getAccountProposals(self, account_ids, callback): - """ Get Account Proposals and call callback + def getObjectscb(self, oids, callback, *args): + # Are they stored in memory already? + if self.objectMap is not None: + for oid in oids: + if oid in self.objectMap and callable(callback): + callback(self.objectMap[oid]) + oids.remove(oid) + # Let's get those that we haven't found in memory! + if oids: + handles = [partial(self.setObjects, oids)] + if callback and callable(callback): + handles.append(callback) + self.wsexec([self.api_ids["database"], + "get_objects", + [oids]], handles) - :param array account_ids: Array containing account ids - :param fnt callback: Callback to execute with the response + def setObject(self, oid, data): + """ Set Object in the internal Object Storage + """ + self.setObjects([oid], [data]) + def setObjects(self, oids, datas): + if self.objectMap is None: + return + + for i, oid in enumerate(oids): + self.objectMap[oid] = datas[i] + + """ Callbacks and dispatcher + """ + def eventcallback(self, name): + """ Call an event callback + + :param str name: Name of the event """ - self.wsexec([self.api_ids["database"], - "get_proposed_transactions", - account_ids], - callback) + if (name in self.onEventCallbacks and + callable(self.onEventCallbacks[name])): + self.onEventCallbacks[name](self) def dispatchNotice(self, notice): """ Main Message Dispatcher for notifications as called by @@ -211,47 +241,35 @@ def dispatchNotice(self, notice): if inst == "1" and _type == "7": for m in self.markets: market = self.markets[m] + if not callable(market["callback"]): + continue if(((market["quote"] == notice["sell_price"]["quote"]["asset_id"] and market["base"] == notice["sell_price"]["base"]["asset_id"]) or (market["base"] == notice["sell_price"]["quote"]["asset_id"] and - market["quote"] == notice["sell_price"]["base"]["asset_id"])) and - callable(market["callback"])): + market["quote"] == notice["sell_price"]["base"]["asset_id"]))): market["callback"](self, notice) - except Exception as e: - print('Error dispatching notice: %s' % str(e)) - import traceback - traceback.print_exc() - - def getObjectcb(self, oid, callback, *args): - """ Get an Object from the internal object storage if available - or otherwise retrieve it from the API. - - :param object-id oid: Object ID to retrieve - :param fnt callback: Callback to call if object has been received - """ - if self.objectMap is not None and oid in self.objectMap and callable(callback): - callback(self.objectMap[oid]) - else: - handles = [partial(self.setObject, oid)] - if callback and callable(callback): - handles.append(callback) - self.wsexec([self.api_ids["database"], - "get_objects", - [[oid]]], handles) + " Asset notifications " + if (inst == "1" and _type == "3" or # Asset itself + # bitasset and dynamic data + inst == "2" and (_type == "4" or _type == "3")): + for asset in self.assets: + if not callable(asset["callback"]): + continue + if (asset.get("id") == notice["id"] or + asset.get("bitasset_data_id", None) == notice["id"] or + asset.get("dynamic_asset_data_id", None) == notice["id"]): + asset["callback"](self, notice) - def setObject(self, oid, data): - """ Set Object in the internal Object Storage - """ - if self.objectMap is not None: - self.objectMap[oid] = data + except Exception as e: + log.error('Error dispatching notice: %s' % str(e)) def onConnect(self, response): """ Is executed on successful connect. Calls event ``connection-init``. """ self.request_id = 1 - print("Server connected: {0}".format(response.peer)) + log.debug("Server connected: {0}".format(response.peer)) self.eventcallback("connection-init") def onOpen(self): @@ -259,7 +277,7 @@ def onOpen(self): requests access to APIs and calls event ``connection-opened``. """ - print("WebSocket connection open.") + log.debug("WebSocket connection open.") self._login() " Register with database " @@ -286,13 +304,12 @@ def onMessage(self, payload, isBinary): payload """ res = json.loads(payload.decode('utf8')) -# print("\n\nServer: " + json.dumps(res,indent=1)) -# print("\n\nServer: " + str(res)) + log.debug(res) if "error" not in res: " Resolve answers from RPC calls " if "id" in res: if res["id"] not in self.requests: - print("Received answer to an unknown request?!") + log.warning("Received answer to an unknown request?!") else: callbacks = self.requests[res["id"]]["callback"] if callable(callbacks): @@ -308,7 +325,7 @@ def onMessage(self, payload, isBinary): [self.dispatchNotice(notice) for notice in res["params"][1][0] if "id" in notice] else: - print("Error! ", res) + log.error("Error! ", res) def setLoop(self, loop): """ Define the asyncio loop so that it can be halted on @@ -320,6 +337,53 @@ def connection_lost(self, errmsg): """ Is called if the connection is lost. Calls event ``connection-closed`` and closes the asyncio main loop. """ - print("WebSocket connection closed: {0}".format(errmsg)) + log.info("WebSocket connection closed: {0}".format(errmsg)) self.loop.stop() self.eventcallback("connection-closed") + + def onClose(self, wasClean, code, reason): + self.connection_lost(reason) + + """ L E G A C Y - C A L L S + """ + def getAccountHistory(self, account_id, callback, + start="1.11.0", stop="1.11.0", limit=100): + """ Get Account history History and call callback + + :param account-id account_id: Account ID to read the history for + :param fnt callback: Callback to execute with the response + :param historyID start: Start of the history (defaults to ``1.11.0``) + :param historyID stop: Stop of the history (defaults to ``1.11.0``) + :param historyID stop: Limit entries by (defaults to ``100``, max ``100``) + :raises ValueError: if the account id is incorrectly formatted + """ + warnings.warn( + "getAccountHistory is deprecated! " + "Use client.ws.get_account_history() instead", + DeprecationWarning + ) + if account_id[0:4] == "1.2." : + self.wsexec([self.api_ids["history"], + "get_account_history", + [account_id, start, 100, stop]], + callback) + else : + raise ValueError("getAccountHistory expects an account" + + "id of the form '1.2.x'!") + + def getAccountProposals(self, account_ids, callback): + """ Get Account Proposals and call callback + + :param array account_ids: Array containing account ids + :param fnt callback: Callback to execute with the response + + """ + warnings.warn( + "getAccountProposals is deprecated! " + "Use client.ws.get_proposed_transactions() instead", + DeprecationWarning + ) + self.wsexec([self.api_ids["database"], + "get_proposed_transactions", + account_ids], + callback) diff --git a/grapheneapi/graphenewsrpc.py b/grapheneapi/graphenewsrpc.py index 41570202..3dd65188 100644 --- a/grapheneapi/graphenewsrpc.py +++ b/grapheneapi/graphenewsrpc.py @@ -2,6 +2,8 @@ from websocket import create_connection import json import time +import logging +log = logging.getLogger(__name__) class RPCError(Exception): @@ -18,6 +20,14 @@ class GrapheneWebsocketRPC(object): :param str url: Websocket URL :param str user: Username for Authentication :param str password: Password for Authentication + :param Array apis: List of APIs to register to (default: ["database", "network_broadcast"]) + + Available APIs + + * database + * network_node + * network_broadcast + * history Usage: @@ -31,50 +41,56 @@ class GrapheneWebsocketRPC(object): subsystem, please use ``GrapheneWebsocket`` instead. """ - call_id = 0 api_id = {} def __init__(self, url, user="", password=""): self.url = url self.user = user self.password = password - self.ws = create_connection(url) - self.login(user, password, api_id=1) + self.wsconnect() + + def wsconnect(self): + while True: + try: + self.ws = create_connection(self.url) + break + except KeyboardInterrupt: + break + except: + log.warning("Cannot connect to WS node: %s" % self.url) + time.sleep(10) + self.login(self.user, self.password, api_id=1) self.api_id["database"] = self.database(api_id=1) self.api_id["history"] = self.history(api_id=1) self.api_id["network_broadcast"] = self.network_broadcast(api_id=1) - # self.enable_pings() - - def _send_ping(self, interval, event): - while not event.wait(interval): - self.last_ping_tm = time.time() - if self.sock: - self.sock.ping() - - def enable_pings(self): - event = threading.Event() - ping_interval = 30 - thread = threading.Thread(target=self._send_ping, args=(ping_interval, event)) - thread.setDaemon(True) - thread.start() - - def get_call_id(self): - """ Get the ID for the next RPC call """ - self.call_id += 1 - return self.call_id def get_account(self, name): + """ Get full account details from account name or id + + :param str name: Account name or account id + """ if len(name.split(".")) == 3: return self.get_objects([name])[0] else : return self.get_account_by_name(name) def get_asset(self, name): + """ Get full asset from name of id + + :param str name: Symbol name or asset id (e.g. 1.3.0) + """ if len(name.split(".")) == 3: return self.get_objects([name])[0] else : return self.lookup_asset_symbols([name])[0] + def get_object(self, o): + """ Get object with id ``o`` + + :param str o: Full object id + """ + return self.get_objects([o])[0] + def getFullAccountHistory(self, account, begin=1, limit=100, sort="block"): """ Get History of an account @@ -165,6 +181,93 @@ def getFullAccountHistory(self, account, begin=1, limit=100, sort="block"): return r + """ Block Streams + """ + def block_stream(self, start=None, mode="irreversible"): + """ Yields blocks starting from ``start``. + + :param int start: Starting block + :param str mode: We here have the choice between + * "head": the last block + * "irreversible": the block that is confirmed by 2/3 of all block producers and is thus irreversible! + """ + # Let's find out how often blocks are generated! + config = self.get_global_properties() + block_interval = config["parameters"]["block_interval"] + + if not start: + props = self.get_dynamic_global_properties() + # Get block number + if mode == "head": + start = props['head_block_number'] + elif mode == "irreversible": + start = props['last_irreversible_block_num'] + else: + raise ValueError( + '"mode" has to be "head" or "irreversible"' + ) + + # We are going to loop indefinitely + while True: + + # Get chain properies to identify the + # head/last reversible block + props = self.get_dynamic_global_properties() + + # Get block number + if mode == "head": + head_block = props['head_block_number'] + elif mode == "irreversible": + head_block = props['last_irreversible_block_num'] + else: + raise ValueError( + '"mode" has to be "head" or "irreversible"' + ) + + # Blocks from start until head block + for blocknum in range(start, head_block + 1): + # Get full block + yield self.get_block(blocknum) + + # Set new start + start = head_block + 1 + + # Sleep for one block + time.sleep(block_interval) + + def stream(self, opName, *args, **kwargs): + """ Yield specific operations (e.g. transfers) only + + :param str opName: Name of the operation, e.g. transfer, + limit_order_create, limit_order_cancel, call_order_update, + fill_order, account_create, account_update, + account_whitelist, account_upgrade, account_transfer, + asset_create, asset_update, asset_update_bitasset, + asset_update_feed_producers, asset_issue, asset_reserve, + asset_fund_fee_pool, asset_settle, asset_global_settle, + asset_publish_feed, witness_create, witness_update, + proposal_create, proposal_update, proposal_delete, + withdraw_permission_create, withdraw_permission_update, + withdraw_permission_claim, withdraw_permission_delete, + committee_member_create, committee_member_update, + committee_member_update_global_parameters, + vesting_balance_create, vesting_balance_withdraw, + worker_create, custom, assert, balance_claim, + override_transfer, transfer_to_blind, blind_transfer, + transfer_from_blind, asset_settle_cancel, asset_claim_fees + :param int start: Begin at this block + """ + from graphenebase.operations import getOperationNameForId + for block in self.block_stream(*args, **kwargs): + if not len(block["transactions"]): + continue + for tx in block["transactions"]: + for op in tx["operations"]: + if getOperationNameForId(op[0]) == opName: + yield op[1] + + """ RPC Calls + """ def rpcexec(self, payload): """ Execute a call by sending the payload @@ -173,8 +276,22 @@ def rpcexec(self, payload): :raises RPCError: if the server returns an error """ try: - self.ws.send(json.dumps(payload)) - ret = json.loads(self.ws.recv()) + log.debug(payload) + while True: + try: + self.ws.send(json.dumps(payload)) + ret = json.loads(self.ws.recv()) + break + except: + log.warning("Cannot connect to WS node: %s" % self.url) + # retry after reconnect + try: + self.ws.close() + self.wsconnect() + except: + pass + log.debug(ret) + if 'error' in ret: if 'detail' in ret['error']: raise RPCError(ret['error']['detail']) diff --git a/graphenebase/account.py b/graphenebase/account.py index d702ae14..a94b7d56 100644 --- a/graphenebase/account.py +++ b/graphenebase/account.py @@ -12,6 +12,38 @@ assert sys.version_info[0] == 3, "graphenelib requires python3" +class PasswordKey(object): + """ This class derives a private key given the account name, the + role and a password. It leverages the technology of Brainkeys + and allows people to have a secure private key by providing a + passphrase only. + """ + + def __init__(self, account, password, role="active"): + self.account = account + self.role = role + self.password = password + + def get_private(self) : + """ Derive private key from the brain key and the current sequence + number + """ + a = bytes(self.account + + self.role + + self.password, 'utf8') + s = hashlib.sha256(a).digest() + return PrivateKey(hexlify(s).decode('ascii')) + + def get_public(self) : + return self.get_private().pubkey + + def get_private_key(self) : + return self.get_private() + + def get_public_key(self) : + return self.get_public() + + class BrainKey(object) : """Brainkey implementation similar to the graphene-ui web-wallet. @@ -39,6 +71,12 @@ def __init__(self, brainkey=None, sequence=0): self.brainkey = self.normalize(brainkey).strip() self.sequence = sequence + def __next__(self): + """ Get the next private key (sequence number increment) for + iterators + """ + return self.next_sequence() + def next_sequence(self) : """ Increment the sequence number by 1 """ self.sequence += 1 diff --git a/graphenebase/operations.py b/graphenebase/operations.py index af595409..7c96f9b3 100644 --- a/graphenebase/operations.py +++ b/graphenebase/operations.py @@ -194,6 +194,7 @@ def __init__(self, *args, **kwargs) : ('extensions', Set([])), ])) + class Call_order_update(GrapheneObject): def __init__(self, *args, **kwargs) : if isArgsThisClass(self, args): @@ -208,3 +209,41 @@ def __init__(self, *args, **kwargs) : ('delta_debt', Asset(kwargs["delta_debt"])), ('extensions', Set([])), ])) + + +class Asset_fund_fee_pool(GrapheneObject): + def __init__(self, *args, **kwargs) : + if isArgsThisClass(self, args): + self.data = args[0].data + else: + if len(args) == 1 and len(kwargs) == 0: + kwargs = args[0] + super().__init__(OrderedDict([ + ('fee', Asset(kwargs["fee"])), + ('from_account', ObjectId(kwargs["from_account"], "account")), + ('asset_id', ObjectId(kwargs["asset_id"], "asset")), + ('amount', Int64(kwargs["amount"])), + ('extensions', Set([])), + ])) + + +class Override_transfer(GrapheneObject) : + def __init__(self, *args, **kwargs) : + if isArgsThisClass(self, args): + self.data = args[0].data + else: + if len(args) == 1 and len(kwargs) == 0: + kwargs = args[0] + if "memo" in kwargs: + memo = Optional(Memo(kwargs["memo"])) + else: + memo = Optional(None) + super().__init__(OrderedDict([ + ('fee' , Asset(kwargs["fee"])), + ('issuer' , ObjectId(kwargs["issuer"], "account")), + ('from' , ObjectId(kwargs["from"], "account")), + ('to' , ObjectId(kwargs["to"], "account")), + ('amount' , Asset(kwargs["amount"])), + ('memo' , memo), + ('extensions', Set([])), + ])) diff --git a/graphenebase/signedtransactions.py b/graphenebase/signedtransactions.py index 7318a302..876095bd 100644 --- a/graphenebase/signedtransactions.py +++ b/graphenebase/signedtransactions.py @@ -16,6 +16,8 @@ from .objects import GrapheneObject, isArgsThisClass from .operations import Operation from .chains import known_chains +import logging +log = logging.getLogger(__name__) class Signed_Transaction(GrapheneObject) : @@ -70,7 +72,7 @@ def derSigToHexSig(self, s) : """ s, junk = ecdsa.der.remove_sequence(unhexlify(s)) if junk : - print('JUNK : %s', hexlify(junk).decode('ascii')) + log.debug('JUNK : %s', hexlify(junk).decode('ascii')) assert(junk == b'') x, s = ecdsa.der.remove_integer(s) y, s = ecdsa.der.remove_integer(s) @@ -183,7 +185,7 @@ def sign(self, wifkeys, chain="STEEM") : while 1 : cnt += 1 if not cnt % 20 : - print("Still searching for a canonical signature. Tried %d times already!" % cnt) + log.info("Still searching for a canonical signature. Tried %d times already!" % cnt) # Deterministic k # @@ -191,7 +193,7 @@ def sign(self, wifkeys, chain="STEEM") : sk.curve.generator.order(), sk.privkey.secret_multiplier, hashlib.sha256, - hashlib.sha256(self.digest + (b'%x' % cnt)).digest()) + hashlib.sha256(self.digest + bytes([cnt])).digest()) # Sign message # diff --git a/graphenebase/test/test_account.py b/graphenebase/test/test_account.py index 45305cec..020154cd 100644 --- a/graphenebase/test/test_account.py +++ b/graphenebase/test/test_account.py @@ -1,6 +1,6 @@ import unittest from graphenebase.base58 import Base58 -from graphenebase.account import BrainKey, Address, PublicKey, PrivateKey +from graphenebase.account import BrainKey, Address, PublicKey, PrivateKey, PasswordKey class Testcases(unittest.TestCase) : @@ -151,7 +151,7 @@ def test_BrainKey_normalize(self): ], [b, b, b, b, b, b, b]) - def test_BrainKey(self): + def test_BrainKey_sequences(self): b = BrainKey("COLORER BICORN KASBEKE FAERIE LOCHIA GOMUTI SOVKHOZ Y GERMAL AUNTIE PERFUMY TIME FEATURE GANGAN CELEMIN MATZO") keys = ["5Hsbn6kXio4bb7eW5bX7kTp2sdkmbzP8kGWoau46Cf7en7T1RRE", "5K9MHEyiSye5iFL2srZu3ZVjzAZjcQxUgUvuttcVrymovFbU4cc", @@ -166,3 +166,30 @@ def test_BrainKey(self): for i in keys: p = b.next_sequence().get_private() self.assertEqual(str(p), i) + + def test_PasswordKey(self): + a = ["Aang7foN3oz1Ungai2qua5toh3map8ladei1eem2ohsh2shuo8aeji9Thoseo7ah", + "iep1Mees9eghiifahwei5iidi0Sazae9aigaeT7itho3quoo2dah5zuvobaelau5", + "ohBeuyoothae5aer9odaegh5Eeloh1fi7obei9ahSh0haeYuas1sheehaiv5LaiX", + "geiQuoo9NeeLoaZee0ain3Ku1biedohsesien4uHo1eib1ahzaesh5shae3iena7", + "jahzeice6Ix8ohBo3eik9pohjahgeegoh9sahthai1aeMahs8ki7Iub1oojeeSuo", + "eiVahHoh2hi4fazah9Tha8loxeeNgequaquuYee6Shoopo3EiWoosheeX6yohg2o", + "PheeCh3ar8xoofoiphoo4aisahjiiPah4vah0eeceiJ2iyeem9wahyupeithah9T", + "IuyiibahNgieshei2eeFu8aic1IeMae9ooXi9jaiwaht4Wiengieghahnguang0U", + "Ipee1quee7sheughemae4eir8pheix3quac3ei0Aquo9ohieLaeseeh8AhGeM2ew", + "Tech5iir0aP6waiMeiHoph3iwoch4iijoogh0zoh9aSh6Ueb2Dee5dang1aa8IiP" + ] + b = ["STM5NyCrrXHmdikC6QPRAPoDjSHVQJe3WC5bMZuF6YhqhSsfYfjhN", + "STM8gyvJtYyv5ZbT2ZxbAtgufQ5ovV2bq6EQp4YDTzQuSwyg7Ckry", + "STM7yE71iVPSpaq8Ae2AmsKfyFxA8pwYv5zgQtCnX7xMwRUQMVoGf", + "STM5jRgWA2kswPaXsQNtD2MMjs92XfJ1TYob6tjHtsECg2AusF5Wo", + "STM6XHwVxcP6zP5NV1jUbG6Kso9m8ZG9g2CjDiPcZpAxHngx6ATPB", + "STM59X1S4ofTAeHd1iNHDGxim5GkLo2AdcznksUsSYGU687ywB5WV", + "STM6BPPL4iSRbFVVN8v3BEEEyDsC1STRK7Ba9ewQ4Lqvszn5J8VAe", + "STM7cdK927wj95ptUrCk6HKWVeF74LG5cTjDTV22Z3yJ4Xw8xc9qp", + "STM7VNFRjrE1hs1CKpEAP9NAabdFpwvzYXRKvkrVBBv2kTQCbNHz7", + "STM7ZZFhEBjujcKjkmY31i1spPMx6xDSRhkursZLigi2HKLuALe5t", + ] + for i, pwd in enumerate(a): + p = format(PasswordKey("xeroc", pwd, "posting").get_public(), "STM") + self.assertEqual(p, b[i]) diff --git a/graphenebase/test/test_transactions.py b/graphenebase/test/test_transactions.py index 1d0a877a..c0b0780c 100644 --- a/graphenebase/test/test_transactions.py +++ b/graphenebase/test/test_transactions.py @@ -4,48 +4,16 @@ from pprint import pprint from binascii import hexlify +prefix = "BTS" +wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" +ref_block_num = 34294 +ref_block_prefix = 3707022213 +expiration = "2016-04-06T08:29:27" -class Testcases(unittest.TestCase) : - - """ - def test_test(self): - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" - - from grapheneapi.grapheneapi import GrapheneAPI - rpc = GrapheneAPI("localhost", 8092) - op = rpc.get_prototype_operation("proposal_update_operation") - op[1]["fee_paying_account"] = "1.2.96086" - op[1]["proposal"] = "1.10.219" - op[1]["active_approvals_to_add"] = ["1.2.96086"] - buildHandle = rpc.begin_builder_transaction() - rpc.add_operation_to_builder_transaction(buildHandle, op) - rpc.set_fees_on_builder_transaction(buildHandle, "1.3.0") - tx = rpc.sign_builder_transaction(buildHandle, False) - compare = rpc.serialize_transaction(tx) - ref_block_num = tx["ref_block_num"] - ref_block_prefix = tx["ref_block_prefix"] - expiration = tx["expiration"] - ops = [transactions.Operation(transactions.Proposal_update(**tx["operations"][0][1]))] - tx = transactions.Signed_Transaction(ref_block_num=ref_block_num, - ref_block_prefix=ref_block_prefix, - expiration=expiration, - operations=ops) - tx = tx.sign([wif], chain=prefix) - txWire = hexlify(bytes(tx)).decode("ascii") - print("\n") - print(txWire[:-130]) - print(compare[:-130]) - self.assertEqual(compare[:-130], txWire[:-130]) - """ +class Testcases(unittest.TestCase) : def test_call_update(self): - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" - ref_block_num = 34294 - ref_block_prefix = 3707022213 - expiration = "2016-04-06T08:29:27" s = {'fee': {'amount': 100, 'asset_id': '1.3.0'}, 'delta_debt': {'amount': 10000, @@ -64,26 +32,17 @@ def test_call_update(self): compare = "f68585abf4dce7c8045701036400000000000000001d00e1f50500000000001027000000000000160000011f2627efb5c5144440e06ff567f1a09928d699ac6f5122653cd7173362a1ae20205952c874ed14ccec050be1c86c1a300811763ef3b481e562e0933c09b40e31fb" self.assertEqual(compare[:-130], txWire[:-130]) - def test_limit_order_create(self): - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" - ref_block_num = 34294 - ref_block_prefix = 3707022213 - expiration = "2016-04-06T08:29:27" - s = {"fee": { - "amount": 100, - "asset_id": "1.3.0" - }, + s = {"fee": {"amount": 100, + "asset_id": "1.3.0" + }, "seller": "1.2.29", - "amount_to_sell": { - "amount": 100000, - "asset_id": "1.3.0" - }, - "min_to_receive": { - "amount": 10000, - "asset_id": "1.3.105" - }, + "amount_to_sell": {"amount": 100000, + "asset_id": "1.3.0" + }, + "min_to_receive": {"amount": 10000, + "asset_id": "1.3.105" + }, "expiration": "2016-05-18T09:22:05", "fill_or_kill": False, "extensions": [] @@ -100,15 +59,9 @@ def test_limit_order_create(self): self.assertEqual(compare[:-130], txWire[:-130]) def test_limit_order_cancel(self): - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" - ref_block_num = 34294 - ref_block_prefix = 3707022213 - expiration = "2016-04-06T08:29:27" - s = {"fee": { - "amount": 0, - "asset_id": "1.3.0" - }, + s = {"fee": {"amount": 0, + "asset_id": "1.3.0" + }, "fee_paying_account": "1.2.104", "order": "1.7.51840", "extensions": [] @@ -124,11 +77,6 @@ def test_limit_order_cancel(self): self.assertEqual(compare[:-130], txWire[:-130]) def test_proposal_update(self): - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" - ref_block_num = 34294 - ref_block_prefix = 3707022213 - expiration = "2016-04-06T08:29:27" s = {'fee_paying_account': "1.2.1", 'proposal': "1.10.90", 'active_approvals_to_add': ["1.2.5"], @@ -146,8 +94,6 @@ def test_proposal_update(self): self.assertEqual(compare[:-130], txWire[:-130]) def test_Transfer(self): - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" pub = format(account.PrivateKey(wif).pubkey, prefix) from_account_id = "1.2.0" to_account_id = "1.2.1" @@ -155,9 +101,6 @@ def test_Transfer(self): asset_id = "1.3.4" message = "abcdefgABCDEFG0123456789" nonce = "5862723643998573708" - ref_block_num = 34294 - ref_block_prefix = 3707022213 - expiration = "2016-04-06T08:29:27" fee = transactions.Asset(amount=0, asset_id="1.3.0") amount = transactions.Asset(amount=int(amount), asset_id=asset_id) @@ -190,13 +133,6 @@ def test_Transfer(self): self.assertEqual(compare[:-130], txWire[:-130]) def test_pricefeed(self): - - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" - ref_block_num = 34294 - ref_block_prefix = 3707022213 - expiration = "2016-04-06T08:29:27" - feed = transactions.PriceFeed(**{ "settlement_price" : transactions.Price( base=transactions.Asset(amount=214211, asset_id="1.3.0"), @@ -229,8 +165,7 @@ def test_pricefeed(self): self.assertEqual(compare[:-130], txWire[:-130]) def test_jsonLoading(self): - - data1 = {"expiration": "2016-04-06T08:29:27", + data1 = {"expiration": expiration, "extensions": [], "operations": [[0, {"amount": {"amount": 1000000, "asset_id": "1.3.4"}, @@ -242,8 +177,8 @@ def test_jsonLoading(self): "nonce": 5862723643998573708, "to": "BTS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"}, "to": "1.2.1"}]], - "ref_block_num": 34294, - "ref_block_prefix": 3707022213, + "ref_block_num": ref_block_num, + "ref_block_prefix": ref_block_prefix, "signatures": ["1f6c1e8df5faf18c3b057ce713ec92f9b487c1ba58138daabc0038741b402c930d63d8d63861740215b4f65eb8ac9185a3987f8239b962181237f47189e21102af"]} a = transactions.Signed_Transaction(data1.copy()) data2 = transactions.JsonObj(a) @@ -263,20 +198,93 @@ def test_jsonLoading(self): for key in check1: self.assertEqual(check1[key], check2[key]) + def test_fee_pool(self): + s = {"fee": {"amount": 10001, + "asset_id": "1.3.0" + }, + "from_account": "1.2.282", + "asset_id": "1.3.32", + "amount": 15557238, + "extensions": [] + } + op = transactions.Asset_fund_fee_pool(**s) + ops = [transactions.Operation(op)] + tx = transactions.Signed_Transaction(ref_block_num=ref_block_num, + ref_block_prefix=ref_block_prefix, + expiration=expiration, + operations=ops) + tx = tx.sign([wif], chain=prefix) + txWire = hexlify(bytes(tx)).decode("ascii") + compare = "f68585abf4dce7c8045701101127000000000000009a02207662ed00000000000000011f39f7dc7745076c9c7e612d40c68ee92d3f4b2696b1838037ce2a35ac259883ba6c6c49d91ad05a7e78d80bb83482c273dbbc911587487bf468b85fb4f537da3d" + self.assertEqual(compare[:-130], txWire[:-130]) + + def test_override_transfer(self): + s = {"fee": {"amount": 0, + "asset_id": "1.3.0"}, + "issuer": "1.2.29", + "from": "1.2.104", + "to": "1.2.29", + "amount": {"amount": 100000, + "asset_id": "1.3.105"}, + "extensions": [] + } + op = transactions.Override_transfer(**s) + ops = [transactions.Operation(op)] + tx = transactions.Signed_Transaction(ref_block_num=ref_block_num, + ref_block_prefix=ref_block_prefix, + expiration=expiration, + operations=ops) + tx = tx.sign([wif], chain=prefix) + txWire = hexlify(bytes(tx)).decode("ascii") + compare = "f68585abf4dce7c8045701260000000000000000001d681da08601000000000069000000012030cc81722c3e67442d2f59deba188f6079c8ba2d8318a642e6a70a125655515f20e2bd3adb2ea886cdbc7f6590c7f8c80818d9176d9085c176c736686ab6c9fd" + self.assertEqual(compare[:-130], txWire[:-130]) + + def compareConstructedTX(self): + # def test_online(self): + # self.maxDiff = None + op = transactions.Override_transfer(**{ + "fee": {"amount": 0, + "asset_id": "1.3.0"}, + "issuer": "1.2.29", + "from": "1.2.104", + "to": "1.2.29", + "amount": {"amount": 100000, + "asset_id": "1.3.105"}, + "extensions": [] + }) -if __name__ == '__main__': - prefix = "BTS" - wif = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3" + ops = [transactions.Operation(op)] + tx = transactions.Signed_Transaction( + ref_block_num=ref_block_num, + ref_block_prefix=ref_block_prefix, + expiration=expiration, + operations=ops + ) + tx = tx.sign([wif], chain=prefix) + txWire = hexlify(bytes(tx)).decode("ascii") from grapheneapi.grapheneapi import GrapheneAPI rpc = GrapheneAPI("localhost", 8092) - tx = rpc.borrow_asset("xeroc", 1, "PEG.FAKEUSD", 1000, False) - print(tx) + compare = rpc.serialize_transaction(transactions.JsonObj(tx)) + print(compare[:-130]) + print(txWire[:-130]) + print(txWire[:-130] == compare[:-130]) + self.assertEqual(compare[:-130], txWire[:-130]) + + def compareNewWire(self): + # def test_online(self): + # self.maxDiff = None + + from grapheneapi.grapheneapi import GrapheneAPI + rpc = GrapheneAPI("localhost", 8092) + tx = rpc.create_account("xeroc", "fsafaasf", "", False) + pprint(tx) compare = rpc.serialize_transaction(tx) ref_block_num = tx["ref_block_num"] ref_block_prefix = tx["ref_block_prefix"] expiration = tx["expiration"] - ops = [transactions.Operation(transactions.Call_order_update(**tx["operations"][0][1]))] + + ops = [transactions.Operation(transactions.Account_create(**tx["operations"][0][1]))] tx = transactions.Signed_Transaction(ref_block_num=ref_block_num, ref_block_prefix=ref_block_prefix, expiration=expiration, @@ -286,5 +294,8 @@ def test_jsonLoading(self): print("\n") print(txWire[:-130]) print(compare[:-130]) + # self.assertEqual(compare[:-130], txWire[:-130]) - print(compare[:-130] == txWire[:-130]) +if __name__ == '__main__': + t = Testcases() + t.compareConstructedTX() diff --git a/grapheneexchange/deep_eq.py b/grapheneexchange/deep_eq.py new file mode 100644 index 00000000..7168fdbc --- /dev/null +++ b/grapheneexchange/deep_eq.py @@ -0,0 +1,35 @@ +def deep_eq(_v1, _v2): + import operator + import types + + def _deep_dict_eq(d1, d2): + k1 = sorted(d1.keys()) + k2 = sorted(d2.keys()) + if k1 != k2: # keys should be exactly equal + return False + return sum(deep_eq(d1[k], d2[k]) for k in k1) == len(k1) + + def _deep_iter_eq(l1, l2): + if len(l1) != len(l2): + return False + return sum(deep_eq(v1, v2) for v1, v2 in zip(l1, l2)) == len(l1) + + op = operator.eq + c1, c2 = (_v1, _v2) + + # guard against strings because they are also iterable + # and will consistently cause a RuntimeError (maximum recursion limit reached) + if isinstance(_v1, str): + return op(c1, c2) + + if isinstance(_v1, dict): + op = _deep_dict_eq + else: + try: + c1, c2 = (list(iter(_v1)), list(iter(_v2))) + except TypeError: + c1, c2 = _v1, _v2 + else: + op = _deep_iter_eq + + return op(c1, c2) diff --git a/grapheneexchange/exchange.py b/grapheneexchange/exchange.py index 7f4b2bc6..69bda3b9 100644 --- a/grapheneexchange/exchange.py +++ b/grapheneexchange/exchange.py @@ -1,10 +1,13 @@ from grapheneapi.grapheneclient import GrapheneClient -from graphenebase import transactions +from graphenebase import transactions, operations from graphenebase.account import PrivateKey from datetime import datetime import time import math from grapheneextra.proposal import Proposal +import logging +from . import deep_eq +log = logging.getLogger(__name__) class NoWalletException(Exception): @@ -161,10 +164,12 @@ def __init__(self, config, **kwargs) : if "prefix" in kwargs: self.prefix = kwargs["prefix"] else: - self.prefix = "BTS" + self.prefix = getattr(config, "prefix", "BTS") #: The wif key can be used for creating transactions **if** not # connected to a cli_wallet + if not hasattr(config, "wif"): + setattr(config, "wif", None) if not getattr(config, "wif"): config.wif = None else: @@ -186,7 +191,6 @@ def __init__(self, config, **kwargs) : )): raise WifNotActive - def formatTimeFromNow(self, secs=0): """ Properly Format Time that is `x` seconds in the future @@ -198,7 +202,21 @@ def formatTimeFromNow(self, secs=0): """ return datetime.utcfromtimestamp(time.time() + int(secs)).strftime('%Y-%m-%dT%H:%M:%S') - def _get_market_name_from_ids(self, quote_id, base_id,) : + def normalizePrice(self, market, price): + """ Because assets have different precisions and orders are + created with a rational price, prices defined in floats will + slightly differ from the actual prices on the blockchain. + This is a representation issuer between floats being + represented as a ratio of integer (satoshis) + """ + m = self._get_assets_from_market(market) + base = m["base"] + quote = m["quote"] + return float( + (int(price * 10 ** (base["precision"] - quote["precision"])) / + 10 ** (base["precision"] - quote["precision"]))) + + def _get_market_name_from_ids(self, quote_id, base_id) : """ Returns the properly formated name of a market given base and quote ids @@ -245,7 +263,7 @@ def _get_assets_from_market(self, market) : """ quote_symbol, base_symbol = market.split(self.market_separator) quote = self.ws.get_asset(quote_symbol) - base = self.ws.get_asset(quote_symbol) + base = self.ws.get_asset(base_symbol) return {"quote" : quote, "base" : base} def _get_price(self, o) : @@ -379,7 +397,6 @@ def returnFees(self) : 'committee_member_create': {'fee': 100000000.0}} """ - from graphenebase.transactions import operations r = {} obj, base = self.ws.get_objects(["2.0.0", "1.3.0"]) fees = obj["parameters"]["current_fees"]["parameters"] @@ -608,11 +625,11 @@ def returnOrderBook(self, currencyPair="all", limit=25): for o in orders: if o["sell_price"]["base"]["asset_id"] == m["base"] : price = self._get_price(o["sell_price"]) - volume = float(o["for_sale"]) / 10 ** quote_asset["precision"] + volume = float(o["for_sale"]) / 10 ** base_asset["precision"] / self._get_price(o["sell_price"]) bids.append([price, volume, o["id"]]) else : price = 1 / self._get_price(o["sell_price"]) - volume = float(o["for_sale"]) / 10 ** quote_asset["precision"] / self._get_price(o["sell_price"]) + volume = float(o["for_sale"]) / 10 ** quote_asset["precision"] asks.append([price, volume, o["id"]]) data = {"asks" : asks, "bids" : bids} @@ -806,7 +823,13 @@ def returnTradeHistory(self, currencyPair="all", limit=25): r.update({market : trades}) return r - def buy(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill=False): + def buy(self, + currencyPair, + rate, + amount, + expiration=7 * 24 * 60 * 60, + killfill=False, + returnID=False): """ Places a buy order in a given market (buy ``quote``, sell ``base`` in market ``quote_base``). Required POST parameters are "currencyPair", "rate", and "amount". If successful, the @@ -817,6 +840,7 @@ def buy(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill= :param number amount: Amount of ``quote`` to buy :param number expiration: (optional) expiration time of the order in seconds (defaults to 7 days) :param bool killfill: flag that indicates if the order shall be killed if it is not filled (defaults to False) + :param bool returnID: If this flag is True, the call will wait for the order to be included in a block and return it's id Prices/Rates are denoted in 'base', i.e. the USD_BTS market is priced in BTS per USD. @@ -837,7 +861,6 @@ def buy(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill= quote_symbol, base_symbol = currencyPair.split(self.market_separator) base = self.ws.get_asset(base_symbol) quote = self.ws.get_asset(quote_symbol) - if self.rpc: transaction = self.rpc.sell_asset(self.config.account, '{:.{prec}f}'.format(amount * rate, prec=base["precision"]), @@ -860,7 +883,8 @@ def buy(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill= "expiration": transactions.formatTimeFromNow(expiration), "fill_or_kill": killfill, } - ops = [transactions.Operation(transactions.Limit_order_create(**s))] + order = transactions.Limit_order_create(**s) + ops = [transactions.Operation(order)] expiration = transactions.formatTimeFromNow(30) ops = transactions.addRequiredFees(self.ws, ops, "1.3.0") ref_block_num, ref_block_prefix = transactions.getBlockParams(self.ws) @@ -871,19 +895,28 @@ def buy(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill= operations=ops ) transaction = transaction.sign([self.config.wif], self.prefix) - transaction = transactions.JsonObj(transaction) + transaction = transactions.JsonObj(transaction) if not (self.safe_mode or self.propose_only): self.ws.broadcast_transaction(transaction, api="network_broadcast") else: raise NoWalletException() - if self.propose_only: - [self.propose_operations.append(o) for o in transaction["operations"]] - return self.propose_operations + if returnID: + return self._waitForOperationsConfirmation(transactions.JsonObj(order)) else: - return transaction - - def sell(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill=False): + if self.propose_only: + [self.propose_operations.append(o) for o in transaction["operations"]] + return self.propose_operations + else: + return transaction + + def sell(self, + currencyPair, + rate, + amount, + expiration=7 * 24 * 60 * 60, + killfill=False, + returnID=False): """ Places a sell order in a given market (sell ``quote``, buy ``base`` in market ``quote_base``). Required POST parameters are "currencyPair", "rate", and "amount". If successful, the @@ -894,6 +927,7 @@ def sell(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill :param number amount: Amount of ``quote`` to sell :param number expiration: (optional) expiration time of the order in seconds (defaults to 7 days) :param bool killfill: flag that indicates if the order shall be killed if it is not filled (defaults to False) + :param bool returnID: If this flag is True, the call will wait for the order to be included in a block and return it's id Prices/Rates are denoted in 'base', i.e. the USD_BTS market is priced in BTS per USD. @@ -936,7 +970,8 @@ def sell(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill "expiration": transactions.formatTimeFromNow(expiration), "fill_or_kill": killfill, } - ops = [transactions.Operation(transactions.Limit_order_create(**s))] + order = transactions.Limit_order_create(**s) + ops = [transactions.Operation(order)] expiration = transactions.formatTimeFromNow(30) ops = transactions.addRequiredFees(self.ws, ops, "1.3.0") ref_block_num, ref_block_prefix = transactions.getBlockParams(self.ws) @@ -947,17 +982,34 @@ def sell(self, currencyPair, rate, amount, expiration=7 * 24 * 60 * 60, killfill operations=ops ) transaction = transaction.sign([self.config.wif], self.prefix) - transaction = transactions.JsonObj(transaction) + transaction = transactions.JsonObj(transaction) if not (self.safe_mode or self.propose_only): self.ws.broadcast_transaction(transaction, api="network_broadcast") else: raise NoWalletException() - if self.propose_only: - [self.propose_operations.append(o) for o in transaction["operations"]] - return self.propose_operations + if returnID: + return self._waitForOperationsConfirmation(transactions.JsonObj(order)) else: - return transaction + if self.propose_only: + [self.propose_operations.append(o) for o in transaction["operations"]] + return self.propose_operations + else: + return transaction + + def _waitForOperationsConfirmation(self, thisop): + if self.safe_mode: + return "Safe Mode enabled, can't obtain an orderid" + counter = -2 + blocknum = int(self.ws.get_dynamic_global_properties()["head_block_number"]) + for block in self.ws.block_stream(start=blocknum - 2, mode="head"): + counter += 1 + for tx in block["transactions"]: + for i, op in enumerate(tx["operations"]): + if deep_eq.deep_eq(op[1], thisop): + return (tx["operation_results"][i][1]) + if counter > 10: + raise Exception("The operation has not been added after 10 blocks!") def list_debt_positions(self): """ List Call Positions (borrowed assets and amounts) @@ -980,11 +1032,17 @@ def list_debt_positions(self): for debt in debts: base = self.getObject(debt["call_price"]["base"]["asset_id"]) quote = self.getObject(debt["call_price"]["quote"]["asset_id"]) - call_price = self._get_price(debt["call_price"]) + + if "bitasset_data_id" not in quote: + continue bitasset = self.getObject(quote["bitasset_data_id"]) settlement_price = self._get_price(bitasset["current_feed"]["settlement_price"]) + if not settlement_price: + continue + + call_price = self._get_price(debt["call_price"]) collateral_amount = int(debt["collateral"]) / 10 ** base["precision"] debt_amount = int(debt["debt"]) / 10 ** quote["precision"] @@ -1522,3 +1580,48 @@ def proposals_clear(self): """ Clear stored proposals """ self.propose_operations = [] + + def fund_fee_pool(self, symbol, amount): + """ Fund the fee pool of an asset with BTS + + :param str symbol: Symbol of the asset to fund + :param float amount: Amount of BTS to use for funding fee pool + """ + if self.safe_mode : + print("Safe Mode enabled!") + print("Please GrapheneExchange(config, safe_mode=False) to remove this and execute the transaction below") + if self.rpc: + transaction = self.rpc.fund_asset_fee_pool(self.config.account, symbol, amount, not (self.safe_mode or self.propose_only)) + elif self.config.wif: + account = self.ws.get_account(self.config.account) + asset = self.ws.get_asset(symbol) + s = {"fee": {"amount": 0, + "asset_id": "1.3.0" + }, + "from_account": account["id"], + "asset_id": asset["id"], + "amount": int(amount * 10 ** asset["precision"]), + "extensions": [] + } + ops = [transactions.Operation(transactions.Asset_fund_fee_pool(**s))] + expiration = transactions.formatTimeFromNow(30) + ops = transactions.addRequiredFees(self.ws, ops, "1.3.0") + ref_block_num, ref_block_prefix = transactions.getBlockParams(self.ws) + transaction = transactions.Signed_Transaction( + ref_block_num=ref_block_num, + ref_block_prefix=ref_block_prefix, + expiration=expiration, + operations=ops + ) + transaction = transaction.sign([self.config.wif], self.prefix) + transaction = transactions.JsonObj(transaction) + if not (self.safe_mode or self.propose_only): + self.ws.broadcast_transaction(transaction, api="network_broadcast") + else: + raise NoWalletException() + + if self.propose_only: + [self.propose_operations.append(o) for o in transaction["operations"]] + return self.propose_operations + else: + return transaction diff --git a/scripts/pricefeeds/config-example.py b/scripts/pricefeeds/config-example.py index 5e8bb5c2..4ef35fc2 100644 --- a/scripts/pricefeeds/config-example.py +++ b/scripts/pricefeeds/config-example.py @@ -64,6 +64,7 @@ _all_assets = ["BTC", "SILVER", "GOLD", "TRY", "SGD", "HKD", "NZD", "CNY", "MXN", "CAD", "CHF", "AUD", "GBP", "JPY", "EUR", "USD", "KRW"] # "SHENZHEN", "HANGSENG", "NASDAQC", "NIKKEI", "RUB", "SEK" + "KRW", "TCNY", "TUSD" ] # "SHENZHEN", "HANGSENG", "NASDAQC", "NIKKEI", "RUB", "SEK" _bases = ["CNY", "USD", "BTC", "EUR", "HKD", "JPY"] asset_config = {"default" : { # DEFAULT BEHAVIOR diff --git a/setup.py b/setup.py index 21005777..78d6aa84 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ ascii = codecs.lookup('ascii') codecs.register(lambda name, enc=ascii: {True: enc}.get(name == 'mbcs')) -VERSION = '0.4' +VERSION = '0.4.1' setup(name='graphenelib', version=VERSION, @@ -37,6 +37,7 @@ "scrypt>=0.7.1", "ecdsa>=0.13", "websocket-client>=0.37.0", + "unidecode", ], classifiers=['License :: OSI Approved :: MIT License', 'Operating System :: OS Independent',