From 59ea71e9431d3e82486c1e0b751d992077291ae4 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Thu, 17 Jan 2019 15:34:56 +0800 Subject: [PATCH 01/23] filter cancelled orders in okexfGateway --- vnpy/api/rest/RestClient.py | 27 +++++++- .../gateway/okexfGateway/okexfGateway.py | 61 ++++++++++++++++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/vnpy/api/rest/RestClient.py b/vnpy/api/rest/RestClient.py index 74ab035..c703dc1 100644 --- a/vnpy/api/rest/RestClient.py +++ b/vnpy/api/rest/RestClient.py @@ -6,6 +6,7 @@ from queue import Empty, Queue from datetime import datetime from multiprocessing.dummy import Pool +from collections import deque import requests from enum import Enum @@ -81,7 +82,9 @@ def __init__(self): self._queue = Queue() self._pool = None # type: Pool - + self._queueing_times = deque(maxlen=100) + self._response_times = deque(maxlen=100) + #---------------------------------------------------------------------- def init(self, urlBase): """ @@ -153,6 +156,9 @@ def addRequest(self, request.extra = extra request.onFailed = onFailed request.onError = onError + request.createDatetime = datetime.now() + request.deliverDatetime = None + request.responseDatetime = None self._queue.put(request) return request @@ -234,14 +240,20 @@ def _processRequest(self, request, session): # type: (Request, requests.Session request = self.sign(request) url = self.makeFullUrl(request.path) - + + request.deliverDatetime = datetime.now() + self._queueing_times.append((request.deliverDatetime - request.createDatetime).total_seconds()) + response = session.request(request.method, url, headers=request.headers, params=request.params, data=request.data) request.response = response - + request.responseDatetime = datetime.now() + + self._response_times.append((request.responseDatetime - request.deliverDatetime).total_seconds()) + httpStatusCode = response.status_code if httpStatusCode / 100 == 2: # 2xx都算成功，尽管交易所都用200 jsonBody = response.json() @@ -273,3 +285,12 @@ def makeFullUrl(self, path): url = self.urlBase + path return url + def getStatus(self): + """ + 获取此时client的一些运行时基本信息 + """ + return { + "queueing_number": self._queue.qsize(), + "avg_queueing_time": sum(self._queueing_times) / len(self._queueing_times) if len(self._queueing_times) else 0, + "avg_response_time": sum(self._response_times) / len(self._response_times) if len(self._response_times) else 0 + } diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index 4f62640..b18e6f3 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -13,6 +13,7 @@ import base64 import zlib from datetime import datetime, timedelta +from collections import OrderedDict from copy import copy from urllib.parse import urlencode import pandas as pd @@ -83,6 +84,7 @@ def connect(self): f.close() try: + debug = str(setting.get("debug", False)) apiKey = str(setting['apiKey']) apiSecret = str(setting['apiSecret']) passphrase = str(setting['passphrase']) @@ -105,6 +107,8 @@ def connect(self): setQryFreq = setting.get('setQryFreq', 60) self.initQuery(setQryFreq) + if debug: + self.startDebugInfo() #---------------------------------------------------------------------- def subscribe(self, subscribeReq): @@ -174,6 +178,23 @@ def startQuery(self): """启动连续查询""" self.eventEngine.register(EVENT_TIMER, self.query) + def startDebugInfo(self): + self.__debug_info_count = 0 + self.eventEngine.register(EVENT_TIMER, self.logDebugInfo) + + def logDebugInfo(self): + self.__debug_info_count += 1 + if self.__debug_info_count > self.qryTrigger: + status = self.restApi.getStatus() + s = ",".join(["%s=%s" % (k, v) for k, v in status.items()]) + content = "RestClient's status: %s" % s + log = VtLogData() + log.gatewayName = self.gatewayName + log.logContent = content + log.logLevel = logging.DEBUG + self.onLog(log) + self.__debug_info_count = 0 + #---------------------------------------------------------------------- def setQryEnabled(self, qryEnabled): """设置是否要启动循环查询""" @@ -299,6 +320,7 @@ def __init__(self, gateway): self.cancelDict = {} self.localRemoteDict = gateway.localRemoteDict self.orderDict = gateway.orderDict + self.cancelledOrders = OrderedDict() #---------------------------------------------------------------------- def sign(self, request): @@ -424,6 +446,9 @@ def sendOrder(self, orderReq):# type: (VtOrderReq)->str def cancelOrder(self, cancelOrderReq): """限速规则：10次/2s""" #symbol = cancelOrderReq.symbol + # avoid repeated order cancelling. + if cancelOrderReq.orderID in self.cancelledOrders: + return orderID = cancelOrderReq.orderID remoteID = self.localRemoteDict.get(orderID, None) print("\ncancelorder\n",remoteID,orderID) @@ -442,8 +467,29 @@ def cancelOrder(self, cancelOrderReq): } path = '/api/futures/v3/cancel_order/%s/%s' %(symbol, remoteID) self.addRequest('POST', path, - callback=self.onCancelOrder)#, - #data=req) + callback=self.onCancelOrder, + onFailed=self.onCancelOrderFailed, + onError=self.onCancelOrderError, + extra=cancelOrderReq, + ) + self._addCancelledOrders(cancelOrderReq.orderID) + + def _addCancelledOrders(self, orderID): + self.cancelledOrders[orderID] = datetime.now() + # avoid memory leak + if len(self.cancelledOrders) >= 10000: + keys = [] + count = 0 + for k in self.cancelledOrder.keys(): + keys.append(k) + count += 1 + if count >= 1000: + break + for k in keys: + self.cancelledOrder.pop(k, None) + + def _removeCancelledOrders(sefl, orderID): + self.cancelledOrders.pop(orderID, None) #---------------------------------------------------------------------- def queryContract(self): @@ -876,6 +922,9 @@ def onCancelOrder(self, data, request): error.errorMsg = str(data['order_id']) + ' ' + ERRORCODE[str(error.errorID)] self.gateway.onError(error) + # only 32004 means this order has already been cancelled or totally executed. + if str(data['error_code']) != '32004': + self._removeCancelledOrders(request.extra.orderID) # could be risky,just testify # if str(data['error_code']) == '32004': # order = self.orderDict.get(str(data['order_id']),None) @@ -890,6 +939,14 @@ def onCancelOrder(self, data, request): # if self.orderDict.get(str(data['order_id']),None): # del self.orderDict[str(data['order_id'])] + def onCancelOrderFailed(self, httpStatusCode, request): + self._removeCancelledOrders(request.extra.orderID) + self.onFailed(httpStatusCode, request) + + def onCancelOrderError(self, exceptionType, exceptionValue, tb, request): + self._removeCancelledOrders(request.extra.orderID) + self.onError(exceptionType, exceptionValue, tb, request) + #---------------------------------------------------------------------- def onFailed(self, httpStatusCode, request): # type:(int, Request)->None """ From 5612eb5badfbbff4f5a930a227dff5cc843163f3 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Thu, 17 Jan 2019 16:28:25 +0800 Subject: [PATCH 02/23] Update okexfGateway.py --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index b18e6f3..a7b7803 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -182,7 +182,7 @@ def startDebugInfo(self): self.__debug_info_count = 0 self.eventEngine.register(EVENT_TIMER, self.logDebugInfo) - def logDebugInfo(self): + def logDebugInfo(self, event): self.__debug_info_count += 1 if self.__debug_info_count > self.qryTrigger: status = self.restApi.getStatus() From d5310bb372bc7330cb8668e6a6ac36b8b1ce8937 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Fri, 18 Jan 2019 10:16:47 +0800 Subject: [PATCH 03/23] fix okexfGateway.py --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index a7b7803..67c45f1 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -480,13 +480,13 @@ def _addCancelledOrders(self, orderID): if len(self.cancelledOrders) >= 10000: keys = [] count = 0 - for k in self.cancelledOrder.keys(): + for k in self.cancelledOrders.keys(): keys.append(k) count += 1 if count >= 1000: break for k in keys: - self.cancelledOrder.pop(k, None) + self.cancelledOrders.pop(k, None) def _removeCancelledOrders(sefl, orderID): self.cancelledOrders.pop(orderID, None) From 0f8586ea33754bdf3fc379aec168946c208b219c Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Fri, 18 Jan 2019 10:31:38 +0800 Subject: [PATCH 04/23] fix okexfGateway 's logDebugInfo --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index 67c45f1..4220f61 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -184,7 +184,7 @@ def startDebugInfo(self): def logDebugInfo(self, event): self.__debug_info_count += 1 - if self.__debug_info_count > self.qryTrigger: + if self.__debug_info_count > 60: status = self.restApi.getStatus() s = ",".join(["%s=%s" % (k, v) for k, v in status.items()]) content = "RestClient's status: %s" % s From daa4ae343e05caf22d5410ed67153c1271dd62cc Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Mon, 21 Jan 2019 21:47:16 +0800 Subject: [PATCH 05/23] change metric file to sqlite --- vnpy/applications/VnObserver/__init__.py | 2 +- vnpy/trader/app/ctaStrategy/plugins/README.md | 6 +- .../ctaStrategy/plugins/ctaMetric/README.md | 10 +- .../app/ctaStrategy/plugins/ctaMetric/base.py | 21 ++- .../plugins/ctaMetric/observers/__init__.py | 7 + .../{observer.py => observers/log.py} | 11 +- .../plugins/ctaMetric/observers/sqlite.py | 120 ++++++++++++++++++ .../plugins/ctaMetric/observers/utils.py | 6 + .../plugins/ctaMetric/senders/__init__.py | 5 + .../ctaMetric/{senders.py => senders/log.py} | 3 +- .../plugins/ctaMetric/senders/sqlite.py | 87 +++++++++++++ 11 files changed, 258 insertions(+), 20 deletions(-) create mode 100644 vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/__init__.py rename vnpy/trader/app/ctaStrategy/plugins/ctaMetric/{observer.py => observers/log.py} (92%) create mode 100644 vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/sqlite.py create mode 100644 vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/utils.py create mode 100644 vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/__init__.py rename vnpy/trader/app/ctaStrategy/plugins/ctaMetric/{senders.py => senders/log.py} (91%) create mode 100644 vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/sqlite.py diff --git a/vnpy/applications/VnObserver/__init__.py b/vnpy/applications/VnObserver/__init__.py index 5fdbb48..9c9639d 100644 --- a/vnpy/applications/VnObserver/__init__.py +++ b/vnpy/applications/VnObserver/__init__.py @@ -1,5 +1,5 @@ import click -from vnpy.trader.app.ctaStrategy.plugins.ctaMetric.observer import run_observer +from vnpy.trader.app.ctaStrategy.plugins.ctaMetric.observers import run_observer app_name="observer" diff --git a/vnpy/trader/app/ctaStrategy/plugins/README.md b/vnpy/trader/app/ctaStrategy/plugins/README.md index 62bc1ff..3dcc239 100644 --- a/vnpy/trader/app/ctaStrategy/plugins/README.md +++ b/vnpy/trader/app/ctaStrategy/plugins/README.md @@ -5,7 +5,7 @@ CtaEngine的核心部分是根据策略的信息(主要是SymbolList)对从Gatew 同样也根据symbol的信息路由到对应的Gateway中执行。同时原本的vnpy还提供了本地Stop单的功能。为了满足不同策略的众多需要，我们需要不断丰富CtaEngine和CtaTemplate的功能。 CtaPlugin就是为了在不改动核心部分且独立解耦地为CtaEngine加入某个新功能而实现而引入的设计。 -> 在此机制下，CtaEngine有了可拓展性，可能被成为更广泛的StrategyEngine比较合适，其他的一些策略引擎和模板比如套利模板也可以在这种机制下用plugin实现。 +> 在此机制下，CtaEngine有了可拓展性，可能被更广泛地称为StrategyEngine比较合适，其他的一些策略引擎和模板比如套利模板也可以在这种机制下用plugin实现。 ## 功能描述 ### CtaEngine中使用的信息 @@ -15,9 +15,9 @@ CtaEngine中的各种处理主要依据以下两种信息： ### CtaEngine本身的功能 CtaEngine本身由两个处理流： - 在Event处理流中加入与策略相关的Tick、Order、Trade、Position等事件的处理，然后视需要转发进策略 -- 对策略调用Gateway中的API接口进行了封装，根据调用时的参数分发到某个具体的Gateway去执行。 +- 对策略调用Gateway中的API接口(主要是下单和撤单)进行了封装，根据调用时的参数分发到某个具体的Gateway去执行。 ### CtaPlugin的功能 -CtaPlugin的功能就是根据CtaEngine中主要用到的信息，在CtaEngine的各处理流的前后设置切面，插入自己的处理逻辑来实现自己的功能。 +CtaPlugin的功能就是根据CtaEngine中主要用到的信息，在CtaEngine的各处理流及Gateway的API接口调用的前后设置切面，插入自己的处理逻辑来实现自己的功能。 具体接口的定义请参考`vnpy.trader.app.ctaStrategy.plugins` > 由于BacktestingEngine本身的实现和接口定义都更复杂，目前各Plugin的BacktestingEngine没有统一框架，需单独实现。 \ No newline at end of file diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/README.md b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/README.md index 217b63e..45d4a53 100644 --- a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/README.md +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/README.md @@ -1,12 +1,12 @@ # Cta策略运行监控插件 ## 情景说明 -不同于国内期货交易，数字货币交易是**7X24小时**的，且提供的交易接口稳定性远不如CTA接口。如果需要在夜晚无人值守时运行策略，需要辅以额外的自动化风控手段，以防止策略运行出现异常甚至运行策略的服务器挂掉的情况带来的不可预计的风险。 +不同于国内期货交易，数字货币交易是**7X24小时**的，且提供的交易接口稳定性远不如CTA接口。如果需要在夜晚无人值守时运行策略，需要辅以外部的自动化风控手段，以防止策略运行出现异常甚至运行策略的服务器挂掉的情况带来的不可预计的风险。 ## 功能描述 该组件用于无人值守运行策略时，监控策略运行时的各项指标，推送到策略外部的监控组件，由外部组件完成风控报警或后续的一些操作，目前选用的外部监控组件为小米开源的[Open-Falcon](https://github.com/open-falcon/falcon-plus)。相对vnpy自带的风控模块主要针对发送订单做事前风控，该组件主要应用场景为事后风控，通过收集的指标也可以辅助对策略运行逻辑进行检查。 推送监控指标的功能由该组件的两部分完成: - `ctaMetricPlugin`跟随策略启动运行，会将最新的监控指标写入一个单独的日志文件中。 -- `ctaMetricObserver`需要在单独启动，observer会监控某根目录下所有记录有监控指标的日志文件，获取到最新的指标值，并按一定频率通过HTTP接口推送给open-falcon +- `ctaMetricObserver`需要在单独启动，observer会监控指定目录下所有记录有监控指标的日志文件，获取到最新的指标值，并按一定频率通过HTTP接口推送给open-falcon 这样的设计旨在尽量减小策略运行时记录监控性能指标的额外耗时，且能在策略进程意外退出时，保持监控指标的推送,从而可以监测到策略意外停止运行的情况。 ## 监控指标 @@ -37,9 +37,9 @@ Open-Falcon中的监控指标，采用和OpenTSDB相似的数据格式：metric | metric | tags | 监控内容 | type | | :-: | :-: | :-: | :-: | -| strategy.heartbeat | `"strategy=%s"%(策略名称)` | 策略心跳，说明策略进程在正常运行并记录监控指标 | COUNTER | -| strategy.trading | `"strategy=%s"%(策略名称)` | 策略是否处于交易状态 | GAUGE | -| gateway.connected | `"strategy=%s,gateway=%s"%(策略名称,gateway名称)` | gateway的连接状态 | GAUGE | +| strategy.heartbeat | `"strategy=%s"%(策略名称)` | 策略心跳，说明策略进程在正常运行并记录监控指标，如果策略正常停止，该项会变为0 | COUNTER | +| strategy.trading | `"strategy=%s"%(策略名称)` | 策略是否处于交易状态,1表示处于交易状态,0表示不处于 | GAUGE | +| gateway.connected | `"strategy=%s,gateway=%s"%(策略名称,gateway名称)` | gateway的连接状态,1表示正常连接,0表示连接方向 | GAUGE | | position.volume | `"strategy=%s,gateway=%s,symbol=%s,direction=%s"%(策略名称,gateway名称,symbol名称,多空方向)` | 按所属策略、交易合约和多空方向分组后，每组的持仓量 | GAUGE | | trade.count | `"strategy=%s,gateway=%s,symbol=%s"%(策略名称,gateway名称,symbol名称)` | 按所属策略和交易合约分组后，每组的成交数 | GAUGE | | trade.volume | `"strategy=%s,gateway=%s,symbol=%s"%(策略名称,gateway名称,symbol名称)` | 按所属策略和交易合约分组后，每组的成交volume | GAUGE | diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/base.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/base.py index 05a3156..5d728fe 100644 --- a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/base.py +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/base.py @@ -49,6 +49,18 @@ def __init__(self): def to_json(self): return json.dumps(self.__dict__, cls=NumpyEncoder) + @classmethod + def from_dict(cls, dct): + obj = cls() + obj.endpoint = dct["endpoint"] + obj.metric = dct["metric"] + obj.timestamp = dct["timestamp"] + obj.step = dct["step"] + obj.value = dct["value"] + obj.counterType = dct["counterType"] + obj.tags = dct["tags"] + return obj + class OpenFalconMetricFactory(object): PREFIX = "vnpy.cta" @@ -254,8 +266,13 @@ def pushMetrics(self): func() except: # prevent stop eventengine's thread self.ctaEngine.error(traceback.format_exc()) - self._metricSender.pushMetrics(self._metricCaches) - # self.ctaEngine.writeCtaLog("推送%s个监控指标" % len(self._metricCaches)) + st = time.time() + try: + self._metricSender.pushMetrics(self._metricCaches) + except: + self.ctaEngine.error(traceback.format_exc()) + et = time.time() + self.ctaEngine.debug("推送%s个监控指标,耗时%s", len(self._metricCaches), et - st) self.clearCache() def clearCache(self): diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/__init__.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/__init__.py new file mode 100644 index 0000000..53409f6 --- /dev/null +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/__init__.py @@ -0,0 +1,7 @@ +from .log import LogFileMetricObserver +from .sqlite import SqliteMetricObserver +from .utils import run_observer as run_observer_cls + +def run_observer(path, url=None, cls=None): + cls = cls or SqliteMetricObserver + run_observer_cls(cls, path, url) diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observer.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/log.py similarity index 92% rename from vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observer.py rename to vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/log.py index 7936d72..f18896b 100644 --- a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observer.py +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/log.py @@ -93,7 +93,7 @@ def on_modified(self, event): self.merge_data(handler) -class MetricFileObserver(object): +class LogFileMetricObserver(object): interval = 10 reg = r".*ctaMetric.log.*" # 监控的文件 @@ -136,10 +136,7 @@ def push_metric(self, df): logging.info("推送%s个指标，response:%s" % (len(payload), r.content)) -def run_observer(path, url=None): - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") - MetricFileObserver(path, url).run() - - if __name__ == "__main__": - run_observer(".") + from utils import run_observer + + run_observer(LogFileMetricObserver, ".") diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/sqlite.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/sqlite.py new file mode 100644 index 0000000..dd52029 --- /dev/null +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/sqlite.py @@ -0,0 +1,120 @@ +import json +import os +import time +import re +import logging + +import requests +from watchdog.observers import Observer +from watchdog.events import RegexMatchingEventHandler +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from vnpy.trader.app.ctaStrategy.plugins.ctaMetric.base import NumpyEncoder +from vnpy.trader.app.ctaStrategy.plugins.ctaMetric.senders.sqlite import OpenFalconMetric as OpenFalconMetricModel +from vnpy.trader.app.ctaStrategy.plugins.ctaMetric.base import OpenFalconMetric + +class HandleMetric(object): + def __init__(self, filepath): + self.filepath = filepath + self._init() + + def _init(self): + engine = create_engine('sqlite:///%s' % self.filepath) + self.engine = engine + Session = sessionmaker() + Session.configure(bind=engine) + self.session = Session() + + def has_table(self): + return self.engine.dialect.has_table(self.engine, OpenFalconMetricModel.__tablename__) + + def get_metrics(self): + if self.has_table(): + metrics = self.session.query(OpenFalconMetricModel).all() + metrics = [OpenFalconMetric.from_dict(metric.__dict__) for metric in metrics] + return metrics + else: + return [] + + +class FileEventHandler(RegexMatchingEventHandler): + def __init__(self, root, reg): + RegexMatchingEventHandler.__init__(self, regexes=[reg]) + self.handlers = {} + self.df_total = None # 所有文件的最后记录的总数据 + self._root = root + self._reg = reg + self._init() + + def _init(self): + for dirpath, _, filenames in os.walk(self._root): + for file in filenames: + if re.match(self._reg, file): + path = os.path.join(dirpath, file) + self.handle_file(path) + + def get_metrics(self): + metrics = {} + for handler in self.handlers.values(): + try: + for metric in handler.get_metrics(): + key = (metric.endpoint, metric.metric) + if key in metrics: + old = metrics[key] + if old.timestamp > metric.timestamp: + continue + metrics[key] = metric + except Exception as e: + logging.exception(e) + continue + return list(metrics.values()) + + def handle_file(self, path): + if path not in self.handlers: + logging.info("开始处理Metric文件: %s" % path) + self.handlers[path] = HandleMetric(path) + + def on_created(self, event): + self.handle_file(event.src_path) + + def on_modified(self, event): + pass + + +class SqliteMetricObserver(object): + interval = 10 + reg = r"ctaMetric.sqlite" # 监控的文件 + + def __init__(self, root=".", url=None): + self._root = os.path.abspath(root) + self._observer = Observer() + self._handler = FileEventHandler(self._root, self.reg) + self._observer.schedule(self._handler, self._root, True) + self._url = url or os.environ.get("OPEN_FALCON_URL", "http://localhost:1988/v1/push") + + def run(self): + self._observer.start() + try: + while True: + time.sleep(self.interval) + self.push_metrics(self._handler.get_metrics()) # 每5秒push一次 + except KeyboardInterrupt: + self._observer.stop() + except Exception as e: + logging.exception(e) + self._observer.join() + + def dump_metrics(self, metrics): + payload = [metric.__dict__ for metric in metrics] + return json.dumps(payload, cls=NumpyEncoder) + + def push_metrics(self, metrics): + r = requests.post(self._url, data=self.dump_metrics(metrics)) + logging.info("推送%s个指标，response:%s" % (len(metrics), r.content)) + + +if __name__ == "__main__": + from utils import run_observer + + run_observer(SqliteMetricObserver, ".") diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/utils.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/utils.py new file mode 100644 index 0000000..6b7b95c --- /dev/null +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/observers/utils.py @@ -0,0 +1,6 @@ + +import logging + +def run_observer(cls, path, url=None): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s") + cls(path, url).run() diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/__init__.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/__init__.py new file mode 100644 index 0000000..a942f4b --- /dev/null +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/__init__.py @@ -0,0 +1,5 @@ +from .log import LogfileMetricSender +from .sqlite import SqliteMetricSender +from ..base import set_sender + +set_sender(SqliteMetricSender) \ No newline at end of file diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/log.py similarity index 91% rename from vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders.py rename to vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/log.py index 09fefdf..d191692 100644 --- a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders.py +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/log.py @@ -6,9 +6,8 @@ from vnpy.trader.utils import Singleton from vnpy.trader.vtFunction import getTempPath -from .base import set_sender, MetricSender +from ..base import MetricSender -@set_sender class LogfileMetricSender(with_metaclass(Singleton, MetricSender)): def __init__(self): super(LogfileMetricSender, self).__init__() diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/sqlite.py b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/sqlite.py new file mode 100644 index 0000000..3e8f9be --- /dev/null +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaMetric/senders/sqlite.py @@ -0,0 +1,87 @@ +import logging +import json +from logging.handlers import TimedRotatingFileHandler + +from six import with_metaclass +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, String, Integer, Float, UniqueConstraint, Index +from sqlalchemy.orm import sessionmaker + +from vnpy.trader.utils import Singleton +from vnpy.trader.vtFunction import getTempPath +from ..base import MetricSender + +Base = declarative_base() + +class OpenFalconMetric(Base): + __tablename__ = 'metrics' + + id = Column(Integer, primary_key=True) + endpoint = Column(String(length=100, convert_unicode=True)) + metric = Column(String(length=100, convert_unicode=True)) + timestamp = Column(Integer()) + step = Column(Integer()) + value = Column(Float()) + counterType = Column(String(length=10, convert_unicode=True)) + tags = Column(String(length=200, convert_unicode=True), nullable=False) + __table_args__ = ( + UniqueConstraint('endpoint', 'metric', name='endpoint_metric_uc'), + Index("endpoint_metric_index", "endpoint", "metric") + ) + + @classmethod + def from_dict(cls, dct): + obj = cls() + obj.endpoint = dct["endpoint"] + obj.metric = dct["metric"] + obj.timestamp = dct["timestamp"] + obj.step = dct["step"] + obj.value = dct["value"] + obj.counterType = dct["counterType"] + obj.tags = dct["tags"] + return obj + + def to_dict(self): + dct = {} + dct["endpoint"] = self.endpoint + dct["metric"] = self.metric + dct["timestamp"] = self.timestamp + dct["step"] = self.step + dct["value"] = self.value + dct["counterType"] = self.counterType + dct["tags"] = self.tags + + +class SqliteMetricSender(with_metaclass(Singleton, MetricSender)): + def __init__(self): + super(SqliteMetricSender, self).__init__() + filename = "ctaMetric.sqlite" + filepath = getTempPath(filename) + engine = create_engine('sqlite:///%s' % filepath) + self.engine = engine + self.ensure_table() + Session = sessionmaker() + Session.configure(bind=engine) + self.session = Session() + + def ensure_table(self): + Base.metadata.create_all(self.engine) + + def pushMetrics(self, metrics): + new = {(metric.endpoint, metric.metric): OpenFalconMetric.from_dict(metric.__dict__) for metric in metrics} + old = {(metric.endpoint, metric.metric): metric for metric in self.session.query(OpenFalconMetric).all()} + objs = [] + for k, v in new.items(): + if k in old: + obj = old[k] + obj.timestamp = v.timestamp + obj.step = v.step + obj.value = v.value + obj.counterType = v.counterType + obj.tags = v.tags + objs.append(obj) + else: + objs.append(v) + self.session.bulk_save_objects(objs) + self.session.commit() From 3f4f29c859c8629ec27fa7d193245bb3e2093c14 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Tue, 22 Jan 2019 10:42:44 +0800 Subject: [PATCH 06/23] fix OkexfGateway._removeCancelledOrders --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index 4220f61..719932e 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -488,7 +488,7 @@ def _addCancelledOrders(self, orderID): for k in keys: self.cancelledOrders.pop(k, None) - def _removeCancelledOrders(sefl, orderID): + def _removeCancelledOrders(self, orderID): self.cancelledOrders.pop(orderID, None) #---------------------------------------------------------------------- From 7bf12b3f1621aa23708782e122a2f624e115799a Mon Sep 17 00:00:00 2001 From: ukamoy Date: Sat, 26 Jan 2019 16:14:35 +0800 Subject: [PATCH 07/23] drop BTG contract | fix tick.lastVolume --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index 719932e..7fd07c7 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -712,6 +712,8 @@ def onQueryContract(self, data, request): # map v1 symbol to contract newcontractDict = {} for newcontract in list(self.contractDict.keys()): + if 'BTG' in newcontract: + break sym = newcontract[:7] if sym in newcontractDict.keys(): newcontractDict[sym].append(newcontract[8:]) @@ -1223,7 +1225,7 @@ def onFuturesTrades(self,d): for n,buf in enumerate(data): tick.lastPrice = float(buf[1]) - tick.lastVolume = float(buf[2]) + tick.lastVolume = int(float(buf[2])/2) tick.lastTradedTime = buf[3] tick.type = buf[4] From b194a31444c0ef16e3c8e972f01db2d43d029771 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Tue, 29 Jan 2019 16:21:41 +0800 Subject: [PATCH 08/23] release dev --- vnpy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/__init__.py b/vnpy/__init__.py index a66a091..3b8ba07 100644 --- a/vnpy/__init__.py +++ b/vnpy/__init__.py @@ -1,4 +1,4 @@ # encoding: UTF-8 -__version__ = '1.1.17' +__version__ = '1.1.17-dev20190129' __author__ = 'Xingetouzi' From 34b7aed8f7588ca583fe2be5264fec7af1cd84f7 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Thu, 14 Feb 2019 12:24:42 +0800 Subject: [PATCH 09/23] fix probable bar offset due to data lack in backtest with barmanager --- vnpy/trader/app/ctaStrategy/plugins/ctaBarManager/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/trader/app/ctaStrategy/plugins/ctaBarManager/manager.py b/vnpy/trader/app/ctaStrategy/plugins/ctaBarManager/manager.py index fa59f69..2ea8291 100644 --- a/vnpy/trader/app/ctaStrategy/plugins/ctaBarManager/manager.py +++ b/vnpy/trader/app/ctaStrategy/plugins/ctaBarManager/manager.py @@ -265,8 +265,8 @@ def _update_with_tick(self, tick, freq): def _update_with_bar(self, bar, freq): current_bar = self._current_bars.get(freq, None) - dt = bar.datetime bt = self._bar_timers[freq] + dt = bt.get_current_dt(bar.datetime) finished_bar = None if current_bar: if bt.is_new_bar(current_bar.datetime, dt): From 82a8af6442fd9986cfc477043b7b6e362a973745 Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Thu, 14 Feb 2019 18:12:57 +0800 Subject: [PATCH 10/23] new features --- vnpy/trader/utils/htmlplot/__init__.py | 6 +- vnpy/trader/utils/htmlplot/core.py | 136 +++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 11 deletions(-) diff --git a/vnpy/trader/utils/htmlplot/__init__.py b/vnpy/trader/utils/htmlplot/__init__.py index cb7bb87..48d6a3b 100644 --- a/vnpy/trader/utils/htmlplot/__init__.py +++ b/vnpy/trader/utils/htmlplot/__init__.py @@ -20,10 +20,10 @@ def showTransaction(engine, frequency="1m", filename=None): if not filename: filename = os.path.join(engine.logPath, "transaction.html") - core.makePlot( + plot = core.makePlot( bars, trades, - filename, frequency ) - + core.output_file(filename) + core.show(plot) diff --git a/vnpy/trader/utils/htmlplot/core.py b/vnpy/trader/utils/htmlplot/core.py index 3600948..8ee3689 100644 --- a/vnpy/trader/utils/htmlplot/core.py +++ b/vnpy/trader/utils/htmlplot/core.py @@ -41,8 +41,8 @@ } ) - -TOOLS.append(hover) +MAINTOOLS = list(TOOLS) +MAINTOOLS.append(hover) def plotCandle(bar, plot, freq=timedelta(minutes=1)): @@ -103,6 +103,31 @@ def plotTradesLine(trades, plot, **kwargs): plot.segment("entryDt", "entryPrice", "exitDt", "exitPrice", source=source, line_dash="dashed", **kwargs) +def plotLine(data, plot, colors=None, index="datetime"): + assert isinstance(plot, Figure) + if isinstance(data, pd.Series): + name = data.name if data.name else "untitled" + data = pd.DataFrame({name: data}) + assert isinstance(data, pd.DataFrame) + if not isinstance(colors, dict): + colors = {} + if index not in data.columns: + if data.index.name != index: + data.index.name = index + data = data.reset_index() + source = ColumnDataSource( + data=data.to_dict("list") + ) + columns = list(data.columns) + columns.remove(index) + for name in columns: + plot.line( + index, name, legend=" %s " % name, color=colors.get(name, None), + source=source + ) + return plot + + def plotTradesTriangle(plot, trades, x, y, size=10, angle_units="deg", **kwargs): assert isinstance(plot, Figure) assert isinstance(trades, pd.DataFrame) @@ -151,12 +176,18 @@ def resample(data, freq): def makeFigure(title="candle"): - plot = figure(x_axis_type="datetime", tools=TOOLS, plot_width=1600, plot_height=800, title=title) + plot = figure(x_axis_type="datetime", tools=MAINTOOLS, plot_width=1600, plot_height=800, title=title) + plot.background_fill_color = properties.background + return plot + + +def makeSubFigure(main_plot, title="", tools=TOOLS, plot_width=1600, plot_height=400): + plot = figure(x_axis_type="datetime", tools=tools, plot_width=plot_width, plot_height=plot_height, x_range=main_plot.x_range) plot.background_fill_color = properties.background return plot -def makePlot(bars, trades, filename="transacton.html", freq=None): +def makePlot(bars, trades, freq=None, plot=None): if isinstance(freq, timedelta): bars = resample(bars.set_index("datetime"), freq).reset_index() freq_name = " ".join(iter_freq(freq)) @@ -164,13 +195,12 @@ def makePlot(bars, trades, filename="transacton.html", freq=None): freq = timedelta(minutes=1) freq_name = "1m" - plot = makeFigure("Transaction | frequency: %s" % freq_name) + if not isinstance(plot, bokeh.plotting.Figure): + plot = makeFigure("Transaction | frequency: %s" % freq_name) plotCandle(bars, plot, freq) plotTrades(trades, plot) - - output_file(filename) - show(plot) + return plot @@ -215,3 +245,93 @@ def freq2timedelta(freq=""): return timedelta(seconds=seconds) +import os + + +class PlotHolder(object): + + def __init__(self, **figure_config): + self.figure_config = figure_config + self.members = [] + self.plot = None + self.tooltips={ + "datetime": "@datetime{%Y-%m-%d %H:%M:%S}", + } + self.formatters={ + "datetime": "datetime", + } + + def add_member(self, _type, params): + if _type == "main": + self.formatters.update({ + "datetime": "datetime", + "entryDt": "datetime", + "exitDt": "datetime" + }) + self.tooltips.update(dict([ + ("datetime", "@datetime{%Y-%m-%d %H:%M:%S}"), + ("open", "@open{0.4f}"), + ("high", "@high{0.4f}"), + ("low", "@low{0.4f}"), + ("close", "@close{0.4f}"), + ("entryDt", "@entryDt{%Y-%m-%d %H:%M:%S}"), + ("entryPrice", "@entryPrice{0.4f}"), + ("exitDt", "@exitDt{%Y-%m-%d %H:%M:%S}"), + ("exitPrice", "@exitPrice{0.4f}"), + ("tradeVolume", "@tradeVolume{0.4f}") + ])) + return + data = params["data"] + + +class MultiPlot(object): + + def __init__(self, filename="BacktestResult.html"): + self.plot_configs = [] + self.plots = [] + self.filename = filename + self.logPath = "" + self._main = None + self.plot_methods = { + "main": self.plot_main + } + + def has_main(self): + return self._main is not None + + def set_main(self, engine, frequency="1m", pos=0): + if isinstance(frequency, str): + frequency = freq2timedelta(frequency) + if not isinstance(frequency, timedelta): + raise TypeError("Type of frequency should be str or datetime.timedelta, not %s" % type(frequency)) + + trade_file = os.path.join(engine.logPath, "交割单.csv") + if not os.path.isfile(trade_file): + raise IOError("Transaction file: %s not exists" % trade_file) + self.logPath = engine.logPath + + trades = read_transaction_file(trade_file) + bars = pd.DataFrame([bar.__dict__ for bar in engine.backtestData]) + frequency = frequency + self.add_plot(pos, "main", trades=trades, bars=bars, frequency=frequency) + + def add_plot(self, pos, _type, **kwargs): + if pos < len(self.plot_configs): + plot_conf = self.plot_configs[pos] + else: + if (len(self.plot_configs)) == 0 and (_type != "main"): + raise ValueError("Should set main plot first.") + plot_conf = [] + self.plot_configs.append(plot_conf) + plot_conf.append({"_type": _type, "params": kwargs}) + + def plot_main(self, bars, trades, frequency=None, plot=None): + plot = makePlot(bars, trades, frequency, plot=plot) + if not self.has_main(): + self._main = plot + return plot + + def plot_line(self, data, colors, plot): + plot = plotLine(data, plot, colors) + return plot + \ No newline at end of file From c257a117b02041661808fc9b9112626891c01965 Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Fri, 15 Feb 2019 17:05:43 +0800 Subject: [PATCH 11/23] MultiPlot --- vnpy/trader/utils/htmlplot/__init__.py | 27 +-- vnpy/trader/utils/htmlplot/core.py | 309 ++++++++++++++++++++----- vnpy/trader/utils/htmlplot/property.py | 9 +- 3 files changed, 267 insertions(+), 78 deletions(-) diff --git a/vnpy/trader/utils/htmlplot/__init__.py b/vnpy/trader/utils/htmlplot/__init__.py index 48d6a3b..66364d9 100644 --- a/vnpy/trader/utils/htmlplot/__init__.py +++ b/vnpy/trader/utils/htmlplot/__init__.py @@ -1,29 +1,16 @@ from datetime import timedelta +from vnpy.trader.utils.htmlplot.core import MultiPlot, read_transaction_file + import pandas as pd import os -def showTransaction(engine, frequency="1m", filename=None): - from vnpy.trader.utils.htmlplot import core - if isinstance(frequency, str): - frequency = core.freq2timedelta(frequency) - if not isinstance(frequency, timedelta): - raise TypeError("Type of frequency should be str or datetime.timedelta, not %s" % type(frequency)) +def showTransaction(engine, frequency="1m", do_resampe=True, filename=None): + mp = MultiPlot.from_engine(engine, frequency, filename=filename) + mp.show() - trade_file = os.path.join(engine.logPath, "交割单.csv") - if not os.path.isfile(trade_file): - raise IOError("Transaction file: %s not exists" % trade_file) - trades = core.read_transaction_file(trade_file) - bars = pd.DataFrame([bar.__dict__ for bar in engine.backtestData]) +def getMultiPlot(engine, freq="1m", do_resampe=True, filename=None): + return MultiPlot.from_engine(engine, freq, do_resampe, filename) - if not filename: - filename = os.path.join(engine.logPath, "transaction.html") - plot = core.makePlot( - bars, - trades, - frequency - ) - core.output_file(filename) - core.show(plot) diff --git a/vnpy/trader/utils/htmlplot/core.py b/vnpy/trader/utils/htmlplot/core.py index 8ee3689..6339451 100644 --- a/vnpy/trader/utils/htmlplot/core.py +++ b/vnpy/trader/utils/htmlplot/core.py @@ -248,90 +248,285 @@ def freq2timedelta(freq=""): import os +DEFAULT_TOOLTIPS = { + "candle": { + "datetime": "@datetime{%Y-%m-%d %H:%M:%S}", + "open": "@open{0.4f}", + "high": "@high{0.4f}", + "low": "@low{0.4f}", + "close": "@close{0.4f}" + }, + "trade": { + "datetime": "@datetime{%Y-%m-%d %H:%M:%S}", + "entryDt": "@entryDt{%Y-%m-%d %H:%M:%S}", + "entryPrice": "@entryPrice{0.4f}", + "exitDt": "@exitDt{%Y-%m-%d %H:%M:%S}", + "exitPrice": "@exitPrice{0.4f}", + "tradeVolume": "@tradeVolume{0.4f}" + } +} + +DEFAULT_FORMATER = { + "trade": { + "entryDt": "datetime", + "exitDt": "datetime" + } +} + +KIND_FORMAT = { + "M": "{%Y-%m-%d %H:%M:%S}", + "f": "{0.4f}", + "O": "", + "i": "" +} + + +PLOT_TYPE = { + "candle": plotCandle, + "line": plotLine, + "trade": plotTrades +} + + +def type2format(dtype): + return KIND_FORMAT.get(dtype.kind, "") + + +import numpy as np + +def random_color(minimum=0): + rbg = [np.random.randint(0, 256) for i in range(3)] + while sum(rbg)1: + holder = self.holders[0] + for i in range(len(self.holders)-1): + if holder.figure_config["plot_height"] > 400: + holder.figure_config["plot_height"] -= 100 + + def add_holder(self, holder): + assert isinstance(holder, PlotHolder) + self.holders.append(holder) + return len(self.holders) - 1 + + def set_main(self, candle, trade, freq="1m", do_resample=True, pos=0): + if pos < len(self.holders): + holder = self.holders[pos] + else: + holder = PlotHolder.main() + pos = self.add_holder(holder) + + if isinstance(freq, str): + freq = freq2timedelta(freq) + + if isinstance(freq, timedelta): + if do_resample: + candle = resample(candle.set_index("datetime"), freq).reset_index() + else: + freq = timedelta(minutes=1) + + holder.add_main_member(candle, trade, freq) + return pos + + def set_engine(self, engine, freq="1m", do_resample=True, pos=0): trade_file = os.path.join(engine.logPath, "交割单.csv") if not os.path.isfile(trade_file): raise IOError("Transaction file: %s not exists" % trade_file) - self.logPath = engine.logPath - trades = read_transaction_file(trade_file) - bars = pd.DataFrame([bar.__dict__ for bar in engine.backtestData]) - frequency = frequency - self.add_plot(pos, "main", trades=trades, bars=bars, frequency=frequency) - - def add_plot(self, pos, _type, **kwargs): - if pos < len(self.plot_configs): - plot_conf = self.plot_configs[pos] + candle = pd.DataFrame([bar.__dict__ for bar in engine.backtestData]) + return self.set_main(candle, trades, freq, do_resample, pos) + + @classmethod + def from_engine(cls, engine, freq="1m", do_resample=True, filename=None): + if not filename: + filename = os.path.join(engine.logPath, "transaction.html") + mp = cls(filename) + mp.set_engine(engine, freq, do_resample) + return mp + + def set_line(self, line, colors=None, pos=None): + return self.set_plot("line", line, pos, colors=colors) + + def set_candle(self, candle, freq="1m", do_resample=True, pos=None): + if isinstance(freq, str): + freq = freq2timedelta(freq) + + if isinstance(freq, timedelta): + if do_resample: + candle = resample(candle.set_index("datetime"), freq).reset_index() + else: + freq = timedelta(minutes=1) + + return self.set_plot("candle", candle, pos, freq=freq) + + def set_plot(self, _type, data, pos=None, **params): + if not isinstance(pos, int): + pos = len(self.holders) + if pos < len(self.holders): + holder = self.holders[pos] else: - if (len(self.plot_configs)) == 0 and (_type != "main"): - raise ValueError("Should set main plot first.") - plot_conf = [] - self.plot_configs.append(plot_conf) - plot_conf.append({"_type": _type, "params": kwargs}) + holder = PlotHolder.sub() + pos = self.add_holder(holder) + holder.add_member(_type, data, **params) + return pos + + def draw_plots(self): + if self.auto_adjust: + self.adjust_figures() + plots = [] + for holder in self.holders: + if self._main: + plot = holder.draw_plot(x_range=self._main.x_range) + else: + plot = holder.draw_plot() + self._main = plot + plots.append(plot) + return plots - def plot_main(self, bars, trades, frequency=None, plot=None): - plot = makePlot(bars, trades, frequency, plot=plot) - if not self.has_main(): - self._main = plot - return plot + def show(self): + plots = self.draw_plots() + output_file(self.filename) + show(column(plots)) - def plot_line(self, data, colors, plot): - plot = plotLine(data, plot, colors) - return plot + def get_data(self, pos, index): + holder = self.holders[pos] + return holder.members[index]["data"].copy() + \ No newline at end of file diff --git a/vnpy/trader/utils/htmlplot/property.py b/vnpy/trader/utils/htmlplot/property.py index e414b71..e7be3a7 100644 --- a/vnpy/trader/utils/htmlplot/property.py +++ b/vnpy/trader/utils/htmlplot/property.py @@ -12,6 +12,12 @@ def __init__(self, **kwargs): self.long_trade_tri = dict() self.short_trade_tri = dict() self.trade_close_tri = dict() + self.default_colors=[ + "#FFFF00", "#FF00FF", "#00FFFF", + "#FF0000", "#00FF00", "#0000FF", + "#C04000", "#C00040", "#00C040", + "#40C000", "#4000C0", "#00f0C0", + ] for key, value in kwargs.items(): setattr(self, key, value) @@ -103,4 +109,5 @@ def __init__(self, **kwargs): ) -# MT4Property --------------------------------------------------------- \ No newline at end of file +# MT4Property --------------------------------------------------------- + From 4e0b1a3d6603f44ff4c815df56a6ee0fd624e657 Mon Sep 17 00:00:00 2001 From: Cam Date: Fri, 15 Feb 2019 19:41:05 +0800 Subject: [PATCH 12/23] add default color --- vnpy/trader/utils/htmlplot/core.py | 15 ++++++++++---- vnpy/trader/utils/htmlplot/property.py | 28 +++++++++++++++++++++----- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/vnpy/trader/utils/htmlplot/core.py b/vnpy/trader/utils/htmlplot/core.py index 6339451..d2cc304 100644 --- a/vnpy/trader/utils/htmlplot/core.py +++ b/vnpy/trader/utils/htmlplot/core.py @@ -482,7 +482,9 @@ def from_engine(cls, engine, freq="1m", do_resample=True, filename=None): return mp def set_line(self, line, colors=None, pos=None): - return self.set_plot("line", line, pos, colors=colors) + holder, pos = self.get_holder(pos) + holder.add_line_member(line, colors) + return pos def set_candle(self, candle, freq="1m", do_resample=True, pos=None): if isinstance(freq, str): @@ -497,6 +499,11 @@ def set_candle(self, candle, freq="1m", do_resample=True, pos=None): return self.set_plot("candle", candle, pos, freq=freq) def set_plot(self, _type, data, pos=None, **params): + holder, pos = self.get_holder(pos) + holder.add_member(_type, data, **params) + return pos + + def get_holder(self, pos): if not isinstance(pos, int): pos = len(self.holders) if pos < len(self.holders): @@ -504,9 +511,9 @@ def set_plot(self, _type, data, pos=None, **params): else: holder = PlotHolder.sub() pos = self.add_holder(holder) - holder.add_member(_type, data, **params) - return pos - + return holder, pos + + def draw_plots(self): if self.auto_adjust: self.adjust_figures() diff --git a/vnpy/trader/utils/htmlplot/property.py b/vnpy/trader/utils/htmlplot/property.py index e7be3a7..395101f 100644 --- a/vnpy/trader/utils/htmlplot/property.py +++ b/vnpy/trader/utils/htmlplot/property.py @@ -12,11 +12,29 @@ def __init__(self, **kwargs): self.long_trade_tri = dict() self.short_trade_tri = dict() self.trade_close_tri = dict() - self.default_colors=[ - "#FFFF00", "#FF00FF", "#00FFFF", - "#FF0000", "#00FF00", "#0000FF", - "#C04000", "#C00040", "#00C040", - "#40C000", "#4000C0", "#00f0C0", + self.default_colors = [ + '#fa8072', + '#f08080', + '#ff0000', + '#ff4500', + '#a52a2a', + '#dc143c', + '#8b0000', + '#c71585', + '#9932cc', + '#800080', + '#4b0082', + '#483d8b', + '#191970', + '#008b8b', + '#2f4f4f', + '#2e8b57', + '#6b8e23', + '#556b2f', + '#ff8c00', + '#a0522d', + '#4682b4', + '#696969' ] for key, value in kwargs.items(): setattr(self, key, value) From 1e34cc14b870c54d7303d78811e70de02f48589c Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Sat, 16 Feb 2019 09:41:45 +0800 Subject: [PATCH 13/23] add vbar --- vnpy/trader/utils/htmlplot/core.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/vnpy/trader/utils/htmlplot/core.py b/vnpy/trader/utils/htmlplot/core.py index 6339451..cd250ea 100644 --- a/vnpy/trader/utils/htmlplot/core.py +++ b/vnpy/trader/utils/htmlplot/core.py @@ -126,7 +126,29 @@ def plotLine(data, plot, colors=None, index="datetime"): source=source ) return plot - + + +def plotVbar(data, plot, colors=None, index="datetime"): + assert isinstance(plot, Figure) + if isinstance(data, pd.Series): + name = data.name if data.name else "untitled" + data = pd.DataFrame({name: data}) + assert isinstance(data, pd.DataFrame) + if not isinstance(colors, dict): + colors = {} + if index not in data.columns: + if data.index.name != index: + data.index.name = index + data = data.reset_index() + data["bottom"] = 0 + source = ColumnDataSource(data=data.to_dict("list")) + for name in data.columns: + if name != index: + plot.vbar( + x="datetime", bottom="bottom", top=name, + legend=" %s " % name, color=colors.get(name, None), + source=source + ) def plotTradesTriangle(plot, trades, x, y, size=10, angle_units="deg", **kwargs): assert isinstance(plot, Figure) @@ -284,7 +306,8 @@ def freq2timedelta(freq=""): PLOT_TYPE = { "candle": plotCandle, "line": plotLine, - "trade": plotTrades + "trade": plotTrades, + "vbar": plotVbar } From 064b6d41b70868361f426ba2f1747ff30202f938 Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Mon, 18 Feb 2019 11:37:16 +0800 Subject: [PATCH 14/23] update htmlplot & optimization --- vnpy/trader/app/ctaStrategy/ctaBacktesting.py | 8 +- vnpy/trader/utils/htmlplot/core.py | 77 ++++++++++--------- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py index 88f5b3d..365d150 100644 --- a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py +++ b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py @@ -1523,6 +1523,11 @@ def addParameter(self, name, start, end=None, step=None): self.paramDict[name] = l + def addParams(self, name, params): + if isinstance(params, str): + params = eval(params) + self.paramDict[name] = list(params) + # ---------------------------------------------------------------------- def generateSetting(self): """生成优化参数组合""" @@ -1634,7 +1639,8 @@ def optimize(backtestEngineClass, strategyClass, setting, targetName, targetValue = d[targetName] except KeyError: targetValue = 0 - return (str(setting), targetValue, d) + # return (str(setting), targetValue, d) + return (setting, targetValue, d) def gen_dates(b_date, days): diff --git a/vnpy/trader/utils/htmlplot/core.py b/vnpy/trader/utils/htmlplot/core.py index 9b14493..61f2c21 100644 --- a/vnpy/trader/utils/htmlplot/core.py +++ b/vnpy/trader/utils/htmlplot/core.py @@ -45,7 +45,7 @@ MAINTOOLS.append(hover) -def plotCandle(bar, plot, freq=timedelta(minutes=1)): +def plotCandle(bar, plot, freq=None): assert isinstance(bar, pd.DataFrame) assert isinstance(plot, Figure) @@ -60,6 +60,8 @@ def plotCandle(bar, plot, freq=timedelta(minutes=1)): )) plot.segment("datetime", "high", "datetime", "low", source=hlsource, **properties.candle_hl) + if not isinstance(freq, timedelta): + freq = bar.datetime.diff().min() width = int(1000*freq.total_seconds()*2/3) incsource = ColumnDataSource(data=dict( @@ -128,7 +130,7 @@ def plotLine(data, plot, colors=None, index="datetime"): return plot -def plotVbar(data, plot, colors=None, index="datetime"): +def plotVbar(data, plot, freq=None, colors=None, index="datetime"): assert isinstance(plot, Figure) if isinstance(data, pd.Series): name = data.name if data.name else "untitled" @@ -140,16 +142,23 @@ def plotVbar(data, plot, colors=None, index="datetime"): if data.index.name != index: data.index.name = index data = data.reset_index() - data["bottom"] = 0 - source = ColumnDataSource(data=data.to_dict("list")) + bottom=pd.Series(0, data.index).values + dct = data.to_dict("list") + dct["_bottom"] = [0] * len(data) + source = ColumnDataSource(data=dct) + if not isinstance(freq, timedelta): + freq = data.datetime.diff().min() + if isinstance(freq, timedelta): + width = int(1000*freq.total_seconds()*2/3) for name in data.columns: if name != index: plot.vbar( - x="datetime", bottom="bottom", top=name, - legend=" %s " % name, color=colors.get(name, None), + x="datetime", bottom="_bottom", top=name, width=width, + legend=" %s " % name, color=colors.get(name, None), alpha=0.5, source=source ) + def plotTradesTriangle(plot, trades, x, y, size=10, angle_units="deg", **kwargs): assert isinstance(plot, Figure) assert isinstance(trades, pd.DataFrame) @@ -165,7 +174,6 @@ def plotTradesTriangle(plot, trades, x, y, size=10, angle_units="deg", **kwargs) plot.triangle(x, y, source=source, size=size, angle_units=angle_units, **kwargs) - def plotTrades(trades, plot=None): assert isinstance(trades, pd.DataFrame) assert isinstance(plot, Figure) @@ -209,23 +217,6 @@ def makeSubFigure(main_plot, title="", tools=TOOLS, plot_width=1600, plot_height return plot -def makePlot(bars, trades, freq=None, plot=None): - if isinstance(freq, timedelta): - bars = resample(bars.set_index("datetime"), freq).reset_index() - freq_name = " ".join(iter_freq(freq)) - else: - freq = timedelta(minutes=1) - freq_name = "1m" - - if not isinstance(plot, bokeh.plotting.Figure): - plot = makeFigure("Transaction | frequency: %s" % freq_name) - - plotCandle(bars, plot, freq) - plotTrades(trades, plot) - return plot - - - def read_transaction_file(filename): trades = pd.read_csv(filename, engine="python") trades["entryDt"] = trades["entryDt"].apply(pd.to_datetime) @@ -241,6 +232,7 @@ def read_transaction_file(filename): ("s", 1) ] + def iter_freq(delta): assert isinstance(delta, timedelta) mod = delta.total_seconds() @@ -394,20 +386,31 @@ def adjust_series(self, data): if data.index.name != "datetime": data.index.name = "datetime" return data.reset_index() + + def adjust_data(self, data): + if isinstance(data, pd.Series): + data = self.adjust_series(data) + assert isinstance(data, pd.DataFrame) + return data - def add_main_member(self, candle, trade, freq="1m"): + def add_main_member(self, candle, trade, freq=None): self.add_member("candle", candle, freq=freq) self.add_member("trade", trade) def add_line_member(self, line, colors=None): - if isinstance(line, pd.Series): - line = self.adjust_series(line) - assert isinstance(line, pd.DataFrame) - + line = self.adjust_data(line) + colors = self.fill_color(colors, line.columns) + self.add_member("line", line, colors=colors) + + def add_vbar_member(self, vbar, freq=None, colors=None): + vbar = self.adjust_data(vbar) + colors = self.fill_color(colors, vbar.columns) + self.add_member("vbar", vbar, colors=colors, freq=freq) + + def fill_color(self, colors, columns): if not isinstance(colors, dict): colors = {} - - for name in line.columns: + for name in columns: if name == "datetime": continue if name not in colors: @@ -416,8 +419,7 @@ def add_line_member(self, line, colors=None): else: color = random_color() colors[name] = color - self.add_member("line", line, colors=colors) - + return colors def make_figure(self, **params): self.figure_config.setdefault( @@ -481,8 +483,6 @@ def set_main(self, candle, trade, freq="1m", do_resample=True, pos=0): if isinstance(freq, timedelta): if do_resample: candle = resample(candle.set_index("datetime"), freq).reset_index() - else: - freq = timedelta(minutes=1) holder.add_main_member(candle, trade, freq) @@ -509,6 +509,10 @@ def set_line(self, line, colors=None, pos=None): holder.add_line_member(line, colors) return pos + def set_vbar(self, data, freq=None, colors=None, pos=None): + holder, pos = self.get_holder(pos) + holder.add_vbar_member(data, freq, colors) + def set_candle(self, candle, freq="1m", do_resample=True, pos=None): if isinstance(freq, str): freq = freq2timedelta(freq) @@ -516,9 +520,6 @@ def set_candle(self, candle, freq="1m", do_resample=True, pos=None): if isinstance(freq, timedelta): if do_resample: candle = resample(candle.set_index("datetime"), freq).reset_index() - else: - freq = timedelta(minutes=1) - return self.set_plot("candle", candle, pos, freq=freq) def set_plot(self, _type, data, pos=None, **params): From cf10f28907ef98656461b0faf69b619c252f0a82 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Tue, 19 Feb 2019 17:56:18 +0800 Subject: [PATCH 15/23] fix potential KeyError related to uncertainty of code executing order in different threads --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index 7fd07c7..0ec911d 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -431,6 +431,9 @@ def sendOrder(self, orderReq):# type: (VtOrderReq)->str order.price = orderReq.price order.totalVolume = orderReq.volume + self.localRemoteDict[orderID] = orderID + self.orderDict[orderID] = order + self.addRequest('POST', '/api/futures/v3/order', callback=self.onSendOrder, data=data, @@ -438,8 +441,6 @@ def sendOrder(self, orderReq):# type: (VtOrderReq)->str onFailed=self.onSendOrderFailed, onError=self.onSendOrderError) - self.localRemoteDict[orderID] = orderID - self.orderDict[orderID] = order return vtOrderID #---------------------------------------------------------------------- From c1923facd0fd9657aa64a4177557f0147c10f66e Mon Sep 17 00:00:00 2001 From: tianrq10 Date: Wed, 20 Feb 2019 17:44:20 +0800 Subject: [PATCH 16/23] fix runOptimization --- vnpy/trader/app/ctaStrategy/ctaBacktesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py index 365d150..d57d6ae 100644 --- a/vnpy/trader/app/ctaStrategy/ctaBacktesting.py +++ b/vnpy/trader/app/ctaStrategy/ctaBacktesting.py @@ -1139,7 +1139,7 @@ def runOptimization(self, strategyClass, optimizationSetting): targetValue = d[targetName] except KeyError: targetValue = 0 - resultList.append(([str(setting)], targetValue, d)) + resultList.append((setting, targetValue, d)) # 显示结果 resultList.sort(reverse=True, key=lambda result: result[1]) From 3adc419f6a30a4267a5d635c9a948ab988556a0d Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Fri, 22 Feb 2019 15:19:20 +0800 Subject: [PATCH 17/23] Readme --- vnpy/trader/utils/htmlplot/Readme.md | 150 +++++++++++++++++++++++++ vnpy/trader/utils/htmlplot/__init__.py | 6 +- vnpy/trader/utils/htmlplot/core.py | 18 ++- 3 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 vnpy/trader/utils/htmlplot/Readme.md diff --git a/vnpy/trader/utils/htmlplot/Readme.md b/vnpy/trader/utils/htmlplot/Readme.md new file mode 100644 index 0000000..aa3db1f --- /dev/null +++ b/vnpy/trader/utils/htmlplot/Readme.md @@ -0,0 +1,150 @@ +# 画图工具 + +htmlplot是为方便策略师分析回测结果提供的画图工具，该工具基于bokeh实现，输出为`.html`文件，可以直接通过浏览器打开。 + +## MultiPlot + +MultiPlot是用于画图的工具类，可以方便调用画出按行排列且横坐标轴绑定的多张时间序列图表，目前支持的图表类型如下： + +* 蜡烛图 +* 线图 +* 柱图 +* 主图 + * 蜡烛图 + * 交割单示意图 + +### 构造方法 + +**`初始化MultiPlot对象`** + +```python +vnpy.trader.utils.htmlplot.core.MultiPlot.__init__(filename="BacktestResult.html", auto_adjust=True) +``` + +|参数名|数据类型|说明| +|:---|:---|:---| +|filename|str|输出文件路径| +|auto_adjust|bool|是否自动调整图表大小| + +**`设置主图`** + +```python +MultiPlot.set_main(candle, trade, freq=None, pos=0) +``` + +|参数名|数据类型|说明| +|:---|:---|:---| +|candle|pandas.DataFrame|K线数据，需要包含列：datetime, open, high, low, close| +|trade|pandas.DataFrame|engine输出的交割单信息，需要包含列：commission, entryDt, entryID, entryPrice, exitDt, exitID, exitPrice, pnl, slippage, turnover, volume| +|freq|None, str, datetime.timedelta|K线周期。`None`: 根据输入数据自动生成; `str`: 数字加周期描述，可用的描述字符：s(second) m(minute) h(hour) d(day); `datetime.timdelta`: 周期长度。例如，4小时线："4h" or datetime.timedelta(hours=4)| +|pos|int, None|图表位置，默认为0。如果大于等于当前图表数或为None则在末尾添加一张新图作为该图的位置。| +|`return`|int|设置的图表编号(pos)。| + + +**`设置K线`** + +```python +MultiPlot.set_candle(candle, freq=None, pos=None) +``` + +|参数名|数据类型|说明| +|:---|:---|:---| +|candle|pandas.DataFrame|K线数据，需要包含列：datetime, open, high, low, close| +|freq|None, str, datetime.timedelta|K线周期。`None`: 根据输入数据自动生成; `str`: 数字加周期描述，可用的描述字符：s(second) m(minute) h(hour) d(day); `datetime.timdelta`: 周期长度。例如，4小时线："4h" or datetime.timedelta(hours=4)| +|pos|int, None|图表位置。如果大于等于当前图表数或为None则在末尾添加一张新图作为该图的位置。| +|`return`|int|设置的图表编号(pos)。| + +**`折线图`** + +```python +MultiPlot.set_line(line, colors=None, pos=None) +``` + +|参数名|数据类型|说明| +|:---|:---|:---| +|line|pandas.DataFrame, pandas.Series|要画线的数据。`pandas.DataFrame` 需要包含列datetime或索引为datetime类型。`pandas.Series`需要有name属性且索引为datetime类型。| +|colors|dict, None|字典中每一个键值对代表对应数据的颜色，如果缺失则会用默认颜色填充。| +|pos|int, None|图表位置。如果大于等于当前图表数或为None则在末尾添加一张新图作为该图的位置。| +|`return`|int|设置的图表编号(pos)。| + + +**`柱图`** +```python +MultiPlot.set_vbar(data, freq=None, colors=None, pos=None) +``` +|参数名|数据类型|说明| +|:---|:---|:---| +|data|pandas.DataFrame, pandas.Series|数据，`pandas.DataFrame` 需要包含列datetime或索引为datetime类型，其中除datetime外每一列代表一列柱。`pandas.Series`需要有name属性且索引为datetime类型。| +|freq|None, str, datetime.timedelta|K线周期。`None`: 根据输入数据自动生成; `str`: 数字加周期描述，可用的描述字符：s(second) m(minute) h(hour) d(day); `datetime.timdelta`: 周期长度。例如，4小时线："4h" or datetime.timedelta(hours=4)| +|colors|dict, None|字典中每一个键值对代表对应数据的颜色，如果缺失则会用默认颜色填充。| +|pos|int, None|图表位置。如果大于等于当前图表数或为None则在末尾添加一张新图作为该图的位置。| +|`return`|int|设置的图表编号(pos)。| + + +**`通过引擎设置主图`** + +```python +MultiPlot.set_engine(engine, freq=None, pos=0) +``` + +|参数名|数据类型|说明| +|:---|:---|:---| +|engine|vnpy.trader.app.ctaStrategy.BacktestingEngine|vnpy回测引擎| +|freq|None, str, datetime.timedelta|K线周期。`None`: 根据输入数据自动生成; `str`: 数字加周期描述，可用的描述字符：s(second) m(minute) h(hour) d(day); `datetime.timdelta`: 周期长度。例如，4小时线："4h" or datetime.timedelta(hours=4)| +|pos|int, None|图表位置，默认为0。如果大于等于当前图表数或为None则在末尾添加一张新图作为该图的位置。| +|`return`|int|设置的图表编号(pos)。| + + +**`通过引擎初始化并设置主图`** + +```python +@classmethod #类方法，调用时不需要初始化，返回值为MultiPlot对象并已经设置好主图0。 +MultiPlot.from_engine(engine, freq=None, filename=None) +``` + +|参数名|数据类型|说明| +|:---|:---|:---| +|engine|vnpy.trader.app.ctaStrategy.BacktestingEngine|vnpy回测引擎| +|freq|None, str, datetime.timedelta|K线周期。`None`: 根据输入数据自动生成; `str`: 数字加周期描述，可用的描述字符：s(second) m(minute) h(hour) d(day); `datetime.timdelta`: 周期长度。例如，4小时线："4h" or datetime.timedelta(hours=4)| +|filename|str|输出文件路径| +|`return`|vnpy.trader.utils.htmlplot.core.MultiPlot|MultiPlot对象| + + +**`画图`** + +```python +MultiPlot.show() +``` +根据设置输出html文件并在默认浏览器中打开。 + + +## 上层函数 + +为方便调用，在htmlplot中提供下列高级函数或属性： + +```python +vnpy.trader.utils.htmlplot.getMultiPlot(engine, freq=None, filename=None) +``` +|参数名|数据类型|说明| +|:---|:---|:---| +|engine|vnpy.trader.app.ctaStrategy.BacktestingEngine|vnpy回测引擎| +|freq|None, str, datetime.timedelta|K线周期。`None`: 根据输入数据自动生成; `str`: 数字加周期描述，可用的描述字符：s(second) m(minute) h(hour) d(day); `datetime.timdelta`: 周期长度。例如，4小时线："4h" or datetime.timedelta(hours=4)| +|filename|str|输出文件路径| +|`return`|vnpy.trader.utils.htmlplot.core.MultiPlot|MultiPlot对象| + + +```python +# 保留老版方法，无返回值直接打开图表. +vnpy.trader.utils.htmlplot.showTransaction(engine, frequency=None, filename=None) +``` +|参数名|数据类型|说明| +|:---|:---|:---| +|engine|vnpy.trader.app.ctaStrategy.BacktestingEngine|vnpy回测引擎| +|freq|None, str, datetime.timedelta|K线周期。`None`: 根据输入数据自动生成; `str`: 数字加周期描述，可用的描述字符：s(second) m(minute) h(hour) d(day); `datetime.timdelta`: 周期长度。例如，4小时线："4h" or datetime.timedelta(hours=4)| +|filename|str|输出文件路径| + + +```python +# MultiPlot类 +vnpy.trader.utils.htmlplot.MultiPlot +``` \ No newline at end of file diff --git a/vnpy/trader/utils/htmlplot/__init__.py b/vnpy/trader/utils/htmlplot/__init__.py index 66364d9..b41ed37 100644 --- a/vnpy/trader/utils/htmlplot/__init__.py +++ b/vnpy/trader/utils/htmlplot/__init__.py @@ -5,12 +5,12 @@ import os -def showTransaction(engine, frequency="1m", do_resampe=True, filename=None): +def showTransaction(engine, frequency=None, filename=None): mp = MultiPlot.from_engine(engine, frequency, filename=filename) mp.show() -def getMultiPlot(engine, freq="1m", do_resampe=True, filename=None): - return MultiPlot.from_engine(engine, freq, do_resampe, filename) +def getMultiPlot(engine, freq=None, filename=None): + return MultiPlot.from_engine(engine, freq, filename) diff --git a/vnpy/trader/utils/htmlplot/core.py b/vnpy/trader/utils/htmlplot/core.py index 61f2c21..0c6561a 100644 --- a/vnpy/trader/utils/htmlplot/core.py +++ b/vnpy/trader/utils/htmlplot/core.py @@ -470,7 +470,7 @@ def add_holder(self, holder): self.holders.append(holder) return len(self.holders) - 1 - def set_main(self, candle, trade, freq="1m", do_resample=True, pos=0): + def set_main(self, candle, trade, freq=None, pos=0): if pos < len(self.holders): holder = self.holders[pos] else: @@ -481,27 +481,26 @@ def set_main(self, candle, trade, freq="1m", do_resample=True, pos=0): freq = freq2timedelta(freq) if isinstance(freq, timedelta): - if do_resample: - candle = resample(candle.set_index("datetime"), freq).reset_index() + candle = resample(candle.set_index("datetime"), freq).reset_index() holder.add_main_member(candle, trade, freq) return pos - def set_engine(self, engine, freq="1m", do_resample=True, pos=0): + def set_engine(self, engine, freq=None, pos=0): trade_file = os.path.join(engine.logPath, "交割单.csv") if not os.path.isfile(trade_file): raise IOError("Transaction file: %s not exists" % trade_file) trades = read_transaction_file(trade_file) candle = pd.DataFrame([bar.__dict__ for bar in engine.backtestData]) - return self.set_main(candle, trades, freq, do_resample, pos) + return self.set_main(candle, trades, freq, pos) @classmethod - def from_engine(cls, engine, freq="1m", do_resample=True, filename=None): + def from_engine(cls, engine, freq=None, filename=None): if not filename: filename = os.path.join(engine.logPath, "transaction.html") mp = cls(filename) - mp.set_engine(engine, freq, do_resample) + mp.set_engine(engine, freq) return mp def set_line(self, line, colors=None, pos=None): @@ -513,13 +512,12 @@ def set_vbar(self, data, freq=None, colors=None, pos=None): holder, pos = self.get_holder(pos) holder.add_vbar_member(data, freq, colors) - def set_candle(self, candle, freq="1m", do_resample=True, pos=None): + def set_candle(self, candle, freq=None, pos=None): if isinstance(freq, str): freq = freq2timedelta(freq) if isinstance(freq, timedelta): - if do_resample: - candle = resample(candle.set_index("datetime"), freq).reset_index() + candle = resample(candle.set_index("datetime"), freq).reset_index() return self.set_plot("candle", candle, pos, freq=freq) def set_plot(self, _type, data, pos=None, **params): From ea96ef18e00ff0da9c02035ff9fbea262b677995 Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Fri, 22 Feb 2019 15:25:19 +0800 Subject: [PATCH 18/23] add deps --- vnpy/trader/utils/htmlplot/Readme.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vnpy/trader/utils/htmlplot/Readme.md b/vnpy/trader/utils/htmlplot/Readme.md index aa3db1f..3f03d78 100644 --- a/vnpy/trader/utils/htmlplot/Readme.md +++ b/vnpy/trader/utils/htmlplot/Readme.md @@ -2,6 +2,10 @@ htmlplot是为方便策略师分析回测结果提供的画图工具，该工具基于bokeh实现，输出为`.html`文件，可以直接通过浏览器打开。 +## 相关依赖 + +* bokeh==0.12.14 + ## MultiPlot MultiPlot是用于画图的工具类，可以方便调用画出按行排列且横坐标轴绑定的多张时间序列图表，目前支持的图表类型如下： From 2f44c69ffa0c4399fe463b8870802c07d956ef3f Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Fri, 22 Feb 2019 15:37:27 +0800 Subject: [PATCH 19/23] vbar pos --- vnpy/trader/utils/htmlplot/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vnpy/trader/utils/htmlplot/core.py b/vnpy/trader/utils/htmlplot/core.py index 0c6561a..e50b244 100644 --- a/vnpy/trader/utils/htmlplot/core.py +++ b/vnpy/trader/utils/htmlplot/core.py @@ -511,6 +511,7 @@ def set_line(self, line, colors=None, pos=None): def set_vbar(self, data, freq=None, colors=None, pos=None): holder, pos = self.get_holder(pos) holder.add_vbar_member(data, freq, colors) + return pos def set_candle(self, candle, freq=None, pos=None): if isinstance(freq, str): From 3121b098cb74f204ac23b030078a0b1c1899bf0b Mon Sep 17 00:00:00 2001 From: caimeng <862786917@qq.com> Date: Mon, 25 Feb 2019 17:37:03 +0800 Subject: [PATCH 20/23] optimize --- vnpy/trader/utils/optimize/Readme.md | 287 +++++++++++++++ vnpy/trader/utils/optimize/__init__.py | 85 +++++ vnpy/trader/utils/optimize/optimization.py | 406 +++++++++++++++++++++ 3 files changed, 778 insertions(+) create mode 100644 vnpy/trader/utils/optimize/Readme.md create mode 100644 vnpy/trader/utils/optimize/__init__.py create mode 100644 vnpy/trader/utils/optimize/optimization.py diff --git a/vnpy/trader/utils/optimize/Readme.md b/vnpy/trader/utils/optimize/Readme.md new file mode 100644 index 0000000..79d4324 --- /dev/null +++ b/vnpy/trader/utils/optimize/Readme.md @@ -0,0 +1,287 @@ +# Optimization + +参数优化工具，重写了相关逻辑，提供更多扩展性。 + +使用方法 + +* 通过optimize模块直接调用。 +* 自定义初始化方式使用。 + + +## Quick Tutorial + +快速使用教程：通过`vnpy.trader.utils.optimize`导入使用。 + +```python +from vnpy.trader.utils import optimize +from hlBreakOutStrategy import hlBreakBtcStrategy +from datetime import datetime + + +def setConfig(root=None): + # 设置策略类 + optimize.strategyClass = hlBreakBtcStrategy + # 设置缓存路径，如果不设置则不会缓存优化结果。 + optimize.root = root + # 设置引擎参数 + optimize.engineSetting = { + "startDate": "20180915 00:00", + "endDate": "20190115 23:59", + "slippage": 0.002, + "rate": 0.0005, + "dbName": "VnTrader_1Min_Db", + } + # 设置策略固定参数 + optimize.globalSetting = { + "symbolList": ["eos.usd.q:okef"], + "barPeriod": 300, + } + # 设置策略优化参数 + optimize.paramsSetting = { + "adxPeriod": range(10, 20, 5), + "adxLowThrehold": range(10, 20, 5) + } + optimize.initOpt() + + + +# 简单优化，无并行，无缓存 +def runSimple(): + start = datetime.now() + print("run simple | start: %s -------------------------------------------" % start) + + setConfig() + + # optimize.run() 在设置好的参数下优化，返回回测结果 + report = optimize.run() + + # Optimization.report 返回优化结果 + print(report) + + end = datetime.now() + print("run simple | end: %s | expire: %s -----------------------------" % (end, end-start)) + + +# 并行优化 无缓存 +def runSimpleParallel(): + start = datetime.now() + print("run simple | start: %s -------------------------------------------" % start) + + setConfig() + # optimize.runParallel() 并行优化，返回回测结果 + report = optimize.runParallel() + print(report) + + end = datetime.now() + print("run simple | end: %s | expire: %s -----------------------------" % (end, end-start)) + + +# 简单优化，无并行，有缓存 +def runMemory(): + + start = datetime.now() + print("run memory | start: %s -------------------------------------------" % start) + + setConfig("test-memory") + # 开始优化，优化返回此次回测结果 + report = optimize.run() + print(report) + + end = datetime.now() + print("run memory | end: %s | expire: %s -----------------------------" % (end, end-start)) + + +# 并行优化，有缓存 +def runMemoryParallel(): + start = datetime.now() + print("run memory | start: %s -------------------------------------------" % start) + + setConfig("test-memory-parallel") + report = optimize.runParallel() + + print(report) + + end = datetime.now() + print("run memory | end: %s | expire: %s -----------------------------" % (end, end-start)) + + +def main(): + # runSimple() + # runSimpleParallel() + # runMemory() + runMemoryParallel() + + +if __name__ == '__main__': + main() +``` + + +## `vnpy.trader.utils.optimize` + +优化器顶层模块，提供了简单设置并优化的方法。 + +### 模块属性 + +|name|type|description| +|:-|:-|:-| +|engineClass|type|回测引擎类，默认为vnpy.trader.app.ctaStrategy.BacktestingEngine。| +|strategyClass|type|策略类，需要继承自vnpy.trader.app.ctaStrategy.CtaTemplate。| +|engineSetting|dict|引擎参数设置。| +|globalSetting|dict|策略固定参数值，包括回测品种等。| +|paramsSetting|dict|优化参数设置，value必须为可迭代对象。| +|root|str, None|文件缓存根目录，如果为None则不使用缓存，默认为None。| + +* engineSetting常用属性 + +|key|type|description|default| +|:-|:-|:-|:-| +|startDate|str|回测开始时间，模式："YYYYmmdd HH:MM"。|无默认值，必填| +|endDate|str|回测结束时间，模式："YYYYmmdd HH:MM"。|无默认值，必填| +|mode|str|回测模式，可选有'tick' 和 'bar'。|'bar'| +|initHours|int|开始时回溯小时数。|0| +|captial|float|回测时的起始本金。|1000000| +|slippage|float|回测时假设的滑点。|0| +|rate|float|回测时假设的佣金比例。|0| +|size|float|合约大小|1| +|priceTick|float|价格最小变动|0| +|dbName|str|回测数据库名|"VnTrader_1Min_Db"| + +* globalSetting常用属性 + +|key|type|description|default| +|:-|:-|:-|:-| +|symbolList|list|回测品种|无默认值，必填| + +### 模块方法 + +```python +initOpt() +``` + +根据模块属性生成并返回优化器。 + +```python +getOpt() +``` + +返回优化器，需要先调用`initOpt()`生成优化器。 + +```python +getMemory() +``` + +返回文件缓存管理器，需要先调用`initOpt()`生成优化器。 + +```python +run() +``` + +开始优化，返回优化结果(pandas.DataFrame)。 + +```python +runParallel() +``` + +并行优化，返回优化结果(pandas.DataFrame)。 + + +## `vnpy.trader.utils.optimize.optimizaion` + +优化器代码主体，主要包括两个类和一些工具方法： + +* `class Optimization` 优化器，提供了扩展方案。 +* `class OptMemory` 优化器文件缓存插件。 +* `def generateSettings` 生成参数组DataFrame。 +* `def frange` 浮点数的range方法。 + +### `vnpy.trader.utils.optimize.optimizaion.Optimizaion` + +构造方法： + +* 初始化 +```python +Optimization.__init__(engineClass, strategyClass, engineSetting, globalSetting, paramsSetting=None) +``` + +|param|type|description| +|:-|:-|:-| +|engineClass|type|回测引擎类，可以设为vnpy.trader.app.ctaStrategy.BacktestingEngine。| +|strategyClass|type|策略类，需要继承自vnpy.trader.app.ctaStrategy.CtaTemplate。| +|engineSetting|dict|引擎参数设置。| +|globalSetting|dict|策略固定参数值，包括回测品种等。| +|paramsSetting|pandas.DataFrame, None|优化参数设置，以行为一次回测的参数组，列为需要回测的参数。| + + +* 通过参数列表初始化 + +```python +# 类方法 +Optimization.generate(engineClass, strategyClass, engineSetting, globalSetting, **params) +``` + +|param|type|description| +|:-|:-|:-| +|engineClass|type|回测引擎类，可以设为vnpy.trader.app.ctaStrategy.BacktestingEngine。| +|strategyClass|type|策略类，需要继承自vnpy.trader.app.ctaStrategy.CtaTemplate。| +|engineSetting|dict|引擎参数设置。| +|globalSetting|dict|策略固定参数值，包括回测品种等。| +|**params|Iterable|优化参数设置，每个值为参数和对应的优化列表。| + + +* 运行优化 + +```python +# 单核优化 +Optimization.run() + +# 并行优化 +Optimization.runParallel() +``` + +* 获取优化结果 + +```python +Optimization.report() +``` + +返回值为pandas.DataFrame。 + +### `vnpy.trader.utils.optimize.optimizaion.OptMemory` + +属性： + +* `optimization` vnpy.trader.utils.optimize.optimizaion.Optimization对象 +* `root` 缓存根目录 +* `results_cache` 优化单次运行结果存储路径 +* `error_cache` 优化错误信息保存路径 +* `index_file` 优化参数索引文件名 +* `result_file` 参数优化结果文件名 + +构造方法： + +* 初始化 +```python +OptMemory.__init__(root=".") +``` + +|param|type|description| +|:-|:-|:-| +|root|str|缓存文件目录，默认为当前目录| + + +* 设置优化器 +```python +OptMemory.generate(engineClass, strategyClass, engineSetting, globalSetting, **params) +``` +生成优化器对象，并向其注册缓存方法。参数与`Optimiztion.generate()`相同。 + + +* 保存并返回优化结果 + +```python +OptMemory.save_report() +``` + + + diff --git a/vnpy/trader/utils/optimize/__init__.py b/vnpy/trader/utils/optimize/__init__.py new file mode 100644 index 0000000..a8bb949 --- /dev/null +++ b/vnpy/trader/utils/optimize/__init__.py @@ -0,0 +1,85 @@ +from vnpy.trader.utils.optimize.optimization import Optimization, OptMemory, frange +from vnpy.trader.app.ctaStrategy import BacktestingEngine, CtaTemplate + + +engineClass = BacktestingEngine +strategyClass = CtaTemplate +engineSetting = { + "dbName": "VnTrader_1Min_Db" +} +globalSetting = {} +paramsSetting = {} +root = None + + +_optimization = None +_memory = None + + +def initOpt(): + assert issubclass(engineClass, BacktestingEngine) + assert issubclass(strategyClass, CtaTemplate) + if isinstance(root, str): + m = OptMemory(root) + m.generate( + engineClass, + strategyClass, + engineSetting, + globalSetting, + **paramsSetting + ) + opt = m.optimization + globals()["_memory"] = m + globals()["_optimization"] = opt + else: + opt = Optimization.generate( + engineClass, + strategyClass, + engineSetting, + globalSetting, + **paramsSetting + ) + globals()["_optimization"] = opt + return opt + + +def setConf(**kwargs): + for key, value in kwargs.items(): + if key not in globals(): + continue + + if isinstance(globals()[key], dict): + assert isinstance(value, dict), "Invalid setting, type of %s should be dict not %s" % (key, type(value)) + globals()[key].update(value) + + else: + globals()[key] = value + + +def getOpt(): + opt = globals()["_optimization"] + assert isinstance(opt, Optimization) + return opt + + +def getMemory(): + m = globals()["_memory"] + assert isinstance(m, OptMemory) + return m + + +def run(): + opt = getOpt().run() + if isinstance(_memory, OptMemory): + return _memory.save_report() + else: + return opt.report() + + +def runParallel(): + opt = getOpt().runParallel() + if isinstance(_memory, OptMemory): + return _memory.save_report() + else: + return opt.report() + diff --git a/vnpy/trader/utils/optimize/optimization.py b/vnpy/trader/utils/optimize/optimization.py new file mode 100644 index 0000000..228a0fa --- /dev/null +++ b/vnpy/trader/utils/optimize/optimization.py @@ -0,0 +1,406 @@ +from vnpy.trader.app.ctaStrategy import BacktestingEngine, CtaTemplate +from vnpy.trader.app.ctaStrategy.ctaBacktesting import optimize +from collections import Iterable +from itertools import product +from json.encoder import JSONEncoder +import pandas as pd +import json +import os +import traceback + + +INDEX_NAME = "_number_" +STATUS = "_status_" + + +def runStrategy(engineClass, strategyClass, engineSetting, globalSetting, strategySetting): + print(engineSetting) + assert issubclass(engineClass, BacktestingEngine) + assert issubclass(strategyClass, CtaTemplate) + + if not isinstance(engineSetting, dict): + engineSetting = {} + + if not isinstance(globalSetting, dict): + globalSetting = {} + + if not isinstance(strategySetting, dict): + strategySetting = {} + + engine = engineClass() + + for key, value in engineSetting.items(): + if hasattr(engine, key): + setattr(engine, key, value) + + engine.setStartDate(engineSetting["startDate"], engineSetting.get("initHours", 0)) + engine.setEndDate(engineSetting["endDate"]) + + engine.initStrategy(strategyClass, {**globalSetting, **strategySetting}) + engine.runBacktesting() + + return engine + + +def runPerformance(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number=0): + engine = runStrategy(engineClass, strategyClass, engineSetting, globalSetting, strategySetting.copy()) + dr = engine.calculateDailyResult() + ds, r = engine.calculateDailyStatistics(dr) + return {"setting": strategySetting, "result": r, INDEX_NAME: number} + + +def runPerformanceParallel(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number=0): + try: + r = runPerformance(engineClass, strategyClass, engineSetting, globalSetting, strategySetting, number) + except: + pe = ParallelError( + number=number, + tb=traceback.format_exc(), + params=strategySetting + ) + raise pe + else: + return r + +class ParallelError(Exception): + + def __init__(self, number, tb, params, *args): + super(ParallelError, self).__init__(number, tb, params, *args) + self.number = number + self.tb = tb + self.params = params + + +class Optimization(object): + + def __init__(self, engineClass, strategyClass, engineSetting, globalSetting, paramsSetting=None): + assert issubclass(engineClass, BacktestingEngine) + assert issubclass(strategyClass, CtaTemplate) + self.engineClass = engineClass + self.strategyClass = strategyClass + self.engineSetting = engineSetting if isinstance(engineSetting, dict) else {} + self.globalSetting = globalSetting if isinstance(globalSetting, dict) else {} + if paramsSetting is None: + self.strategySettings = None + self.paramNames = [] + else: + self.initSettings(paramsSetting) + + self._results = {} + self.errors = [] + + self._callbacks = [self._callback] + self._e_callbacks = [self._error_callback] + + @property + def ready(self): + if not isinstance(self.strategySettings, pd.DataFrame): + return False + + if not self.paramNames: + return False + + return True + + @property + def finished(self): + if not self.ready: + raise ValueError("Optimizaion setting not correct.") + for value in self.strategySetting[STATUS]: + if not value: + return False + return True + + @classmethod + def generate(cls, engineClass, strategyClass, engineSetting, globalSetting, **params): + return cls(engineClass, strategyClass, engineSetting, globalSetting, generateSettings(**params)) + + def initSettings(self, strategySettings=None): + assert isinstance(strategySettings, pd.DataFrame) + self.strategySettings = strategySettings.copy() + self.paramNames = list(self.strategySettings.columns) + if STATUS in self.paramNames: + self.paramNames.remove(STATUS) + else: + self.strategySettings[STATUS] = 0 + self.strategySettings.index.name=INDEX_NAME + + def fill_index(self, keys): + self.strategySettings.loc[keys] = 1 + + def _callback(self, result): + index = result.pop(INDEX_NAME) + self._results[index] = result + self.strategySettings.loc[index, STATUS] = 1 + + def addCallback(self, callback): + self._callbacks.append(callback) + + def addErrorCallback(self, eCallback): + self._e_callbacks.append(eCallback) + + def callback(self, result): + for func in self._callbacks: + func(result.copy()) + + def error_callback(self, error): + for e_callback in self._e_callbacks: + e_callback(error) + + def _error_callback(self, error): + self.errors.append(error) + if isinstance(error, ParallelError): + print("-"*40, "error", "-"*40) + print("number: ", error.number) + print(error.params) + print(error.tb) + print("-"*40, "error", "-"*40) + else: + print(error) + + def iter_settings(self): + return self.strategySettings[self.strategySettings[STATUS]==0][self.paramNames].iterrows() + + def run(self): + if not self.ready: + return self + + for index, strategySetting in self.iter_settings(): + try: + result = runPerformance( + self.engineClass, + self.strategyClass, + self.engineSetting.copy(), + self.globalSetting, + strategySetting.to_dict(), + index + ) + except Exception as e: + import traceback + traceback.print_exc() + self.error_callback(e) + else: + self.callback(result) + + return self + + def runParallel(self): + if not self.ready: + return self + + import multiprocessing + + pool = multiprocessing.Pool() + for index, strategySetting in self.iter_settings(): + pool.apply_async( + runPerformanceParallel, + (self.engineClass, self.strategyClass, self.engineSetting, self.globalSetting, strategySetting.to_dict(), index), + callback=self.callback, + error_callback=self.error_callback + ) + pool.close() + pool.join() + return self + + def clear(self): + self._results = [] + self.errors = [] + + def report(self): + docs = [] + for index, result in self.results.items(): + if "error" in result: + continue + r = result["result"] + + docs.append({INDEX_NAME: index, **r}) + if len(docs): + results = pd.DataFrame(docs).set_index(INDEX_NAME) + return pd.concat([self.strategySettings, results], axis=1).reindex(results.index) + else: + return pd.DataFrame() + + @property + def results(self): + return self._results + + +class OptJsonEncoder(JSONEncoder): + + def default(self, o): + if hasattr(o, "dtype") and o.dtype.kind == "i": + return int(o) + else: + return JSONEncoder.default(self, o) + + +class OptMemory(object): + + def __init__(self, root="."): + self.root = root + if not os.path.isdir(self.root): + os.makedirs(self.root) + self.results_cache = os.path.join(self.root, "opt-cache") + self.error_cache = os.path.join(self.root, "error-cache") + for path in [self.results_cache, self.error_cache]: + if not os.path.isdir(path): + os.makedirs(path) + self.index_file = os.path.join(self.root, "params.csv") + self.result_file = os.path.join(self.root, "report.csv") + self.optimization = None + + def generate(self, engineClass, strategyClass, engineSetting, globalSetting, **params): + if os.path.isfile(self.index_file): + paramsSetting = self.read(self.index_file) + else: + paramsSetting = generateSettings(**params) + opt = Optimization(engineClass, strategyClass, engineSetting, globalSetting, paramsSetting) + self.setOpt(opt) + return self + + def read(self, filename): + assert os.path.isfile(filename), "%s doesn't exist." % filename + try: + return pd.read_csv(filename, index_col=INDEX_NAME) + except Exception as e: + restore(filename) + return pd.read_csv(filename, index_col=INDEX_NAME) + + def setOpt(self, optimization): + assert isinstance(optimization, Optimization) + self.optimization = optimization + self.optimization.addCallback(self.callback) + self.optimization.addErrorCallback(self.error_callback) + if not os.path.isfile(self.index_file): + self.flush_index() + + def callback(self, result): + index = result[INDEX_NAME] + filename = os.path.join(self.results_cache, "%d.json" % index) + result["result"].pop("startDate", None) + result["result"].pop("endDate", None) + with open(filename, "w") as f: + json.dump(result, f, cls=OptJsonEncoder) + + def error_callback(self, error): + if isinstance(error, ParallelError): + r = {} + r["params"] = error.params + r["traceback"] = error.tb + filename = os.path.join(self.error_cache, "%d.json" % error.number) + with open(filename, "w") as f: + json.dump(r, f, cls=JSONEncoder) + + + def flush_index(self): + if isinstance(self.optimization, Optimization): + self.flush(self.index_file, self.optimization.strategySettings) + + def fill_index(self): + for index in self.optimization.strategySettings[self.optimization.strategySettings[STATUS]==0].index: + filename = os.path.join(self.results_cache, "%d.json" % index) + if os.path.isfile(filename): + self.optimization.fill_index(index) + self.flush_index() + + def save_report(self): + report = self.optimization.report() + + report = pd.concat([report, self.read_result()]) + + results = [] + for index in self.optimization.strategySettings[self.optimization.strategySettings[STATUS]==0].index: + filename = os.path.join(self.results_cache, "%d.json" % index) + self.add_result(filename, results) + + if results: + cr = pd.DataFrame(results).set_index(INDEX_NAME) + self.optimization.fill_index(cr.index) + df = pd.concat([ + self.optimization.strategySettings.reindex(cr.index), + cr + ], axis=1) + report = pd.concat([ + report, + df + ]) + self.flush_result(report) + self.flush_index() + return report + + def flush(self, filename, table): + backup(filename) + table.to_csv(filename) + + def flush_result(self, result): + table = self.read_result() + if len(table): + result = pd.concat( + [table, result] + ) + result = result[~result.index.duplicated(keep="last")] + self.flush(self.result_file, result) + + def read_result(self): + if os.path.isfile(self.result_file): + return pd.read_csv(self.result_file, index_col=INDEX_NAME) + else: + table = pd.DataFrame() + table.index.name = INDEX_NAME + return table + + @staticmethod + def add_result(filename, results): + if not os.path.isfile(filename): + return + with open(filename) as f: + try: + result = json.load(f) + results.append({INDEX_NAME: result[INDEX_NAME], **result["result"]}) + except: + pass + +import shutil + + +def bak_name(filename): + root, fname = os.path.split(filename) + if "." in fname: + name, ftype = fname.rsplit(".", 1) + return os.path.join(root, ".".join([name, "bak", ftype])) + else: + return filename+".bak" + + +def backup(filename): + if os.path.isfile(filename): + shutil.copy(filename, bak_name(filename)) + + +def restore(filename): + bfilename = bak_name(filename) + if os.path.isfile(bfilename): + shutil.copy(bfilename, filename) + + +def generateSettings(**params): + keys, values = [], [] + for key, value in list(params.items()): + if not isinstance(value, Iterable): + value = [value] + keys.append(key) + values.append(value) + return pd.DataFrame(list(product(*values)), columns=keys) + + +def frange(start, stop, step): + m = 1 + while step % 1: + m *= 10 + step *= 10 + start *= m + stop *= m + while start < stop: + yield start / m + start += step + From cf9fda0c27440848728001790c6d38b764c999d0 Mon Sep 17 00:00:00 2001 From: ukamoy Date: Mon, 25 Feb 2019 18:05:35 +0800 Subject: [PATCH 21/23] fix oid --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index 7fd07c7..03db957 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -401,7 +401,7 @@ def writeLog(self, content): def sendOrder(self, orderReq):# type: (VtOrderReq)->str """限速规则：20次/2s""" self.orderID += 1 - orderID = str(self.loginTime + self.orderID) + orderID = "FUTURE" + str(self.loginTime + self.orderID) vtOrderID = VN_SEPARATOR.join([self.gatewayName, orderID]) type_ = typeMap[(orderReq.direction, orderReq.offset)] @@ -817,7 +817,7 @@ def onQueryOrder(self, data, request): order.vtSymbol = VN_SEPARATOR.join([order.symbol, order.gatewayName]) self.orderID += 1 - order.orderID = str(self.loginTime + self.orderID) + order.orderID = "FUTURE" + str(self.loginTime + self.orderID) order.vtOrderID = VN_SEPARATOR.join([self.gatewayName, order.orderID]) self.localRemoteDict[order.orderID] = d['order_id'] order.tradedVolume = 0 @@ -1261,7 +1261,7 @@ def onTrade(self, d): else: restApi = self.gateway.restApi restApi.orderID += 1 - order.orderID = str(restApi.loginTime + restApi.orderID) + order.orderID = "FUTURE"+str(restApi.loginTime + restApi.orderID) order.vtOrderID = VN_SEPARATOR.join([self.gatewayName, order.orderID]) order.price = data['price'] From 3c32cce19c9b950fa9751ad645cf45ded32d2623 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Tue, 26 Feb 2019 10:21:55 +0800 Subject: [PATCH 22/23] extract function getOrderID in restApi --- vnpy/trader/gateway/okexfGateway/okexfGateway.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/vnpy/trader/gateway/okexfGateway/okexfGateway.py b/vnpy/trader/gateway/okexfGateway/okexfGateway.py index 03db957..47097e4 100644 --- a/vnpy/trader/gateway/okexfGateway/okexfGateway.py +++ b/vnpy/trader/gateway/okexfGateway/okexfGateway.py @@ -322,6 +322,10 @@ def __init__(self, gateway): self.orderDict = gateway.orderDict self.cancelledOrders = OrderedDict() + #---------------------------------------------------------------------- + def getOrderID(self): + return "FUTURE" + str(self.loginTime + self.orderID) + #---------------------------------------------------------------------- def sign(self, request): """okex的签名方案""" @@ -401,7 +405,7 @@ def writeLog(self, content): def sendOrder(self, orderReq):# type: (VtOrderReq)->str """限速规则：20次/2s""" self.orderID += 1 - orderID = "FUTURE" + str(self.loginTime + self.orderID) + orderID = self.getOrderID() vtOrderID = VN_SEPARATOR.join([self.gatewayName, orderID]) type_ = typeMap[(orderReq.direction, orderReq.offset)] @@ -817,7 +821,7 @@ def onQueryOrder(self, data, request): order.vtSymbol = VN_SEPARATOR.join([order.symbol, order.gatewayName]) self.orderID += 1 - order.orderID = "FUTURE" + str(self.loginTime + self.orderID) + order.orderID = self.getOrderID() order.vtOrderID = VN_SEPARATOR.join([self.gatewayName, order.orderID]) self.localRemoteDict[order.orderID] = d['order_id'] order.tradedVolume = 0 @@ -1261,7 +1265,7 @@ def onTrade(self, d): else: restApi = self.gateway.restApi restApi.orderID += 1 - order.orderID = "FUTURE"+str(restApi.loginTime + restApi.orderID) + order.orderID = restApi.getOrderID() order.vtOrderID = VN_SEPARATOR.join([self.gatewayName, order.orderID]) order.price = data['price'] From 5c9b0ede4c39a12a2fe04fb99a5d895713000102 Mon Sep 17 00:00:00 2001 From: BurdenBear <541795600@qq.com> Date: Tue, 26 Feb 2019 10:39:48 +0800 Subject: [PATCH 23/23] update version to 1.1.18 --- vnpy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnpy/__init__.py b/vnpy/__init__.py index 3b8ba07..8631311 100644 --- a/vnpy/__init__.py +++ b/vnpy/__init__.py @@ -1,4 +1,4 @@ # encoding: UTF-8 -__version__ = '1.1.17-dev20190129' +__version__ = '1.1.18' __author__ = 'Xingetouzi'