From c34357b90fed774d4da1c098b5ea87571360115e Mon Sep 17 00:00:00 2001 From: holger krekel Date: Thu, 5 May 2022 13:47:08 +0200 Subject: [PATCH] Introduce caching of configured live accounts in each test process --- python/src/deltachat/testplugin.py | 88 ++++++++++++++++++++++++++++-- python/tests/test_4_lowlevel.py | 14 ++++- 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/python/src/deltachat/testplugin.py b/python/src/deltachat/testplugin.py index 7ccbe42793..c07a8d5187 100644 --- a/python/src/deltachat/testplugin.py +++ b/python/src/deltachat/testplugin.py @@ -13,6 +13,7 @@ import pytest import requests +import py from . import Account, const, account_hookimpl, get_core_info from .events import FFIEventLogger, FFIEventTracker @@ -178,6 +179,50 @@ def get_liveconfig_producer(self): yield config pytest.fail("more than {} live accounts requested.".format(MAX_LIVE_CREATED_ACCOUNTS)) + def cache_maybe_retrieve_configured_db_files(self, cache_addr, db_target_path): + db_target_path = py.path.local(db_target_path) + assert not db_target_path.exists() + + print("checking cache for", cache_addr) + try: + filescache = self._addr2files[cache_addr] + except KeyError: + print("CACHE FAIL for", cache_addr) + return False + else: + print("CACHE HIT for", cache_addr) + targetdir = db_target_path.dirpath() + write_dict_to_dir(filescache, targetdir) + return True + + def cache_maybe_store_configured_db_files(self, acc): + addr = acc.get_config("addr") + assert acc.is_configured() + # don't overwrite existing entries + if addr not in self._addr2files: + print("storing cache for", addr) + basedir = py.path.local(acc.get_blobdir()).dirpath() + self._addr2files[addr] = create_dict_from_files_in_path(basedir) + return True + + +def create_dict_from_files_in_path(path): + base = py.path.local(path) + cachedict = {} + for path in base.visit(fil=py.path.local.isfile): + cachedict[path.relto(base)] = path.read_binary() + return cachedict + + +def write_dict_to_dir(dic, target_dir): + assert dic + target_dir = py.path.local(target_dir) + for relpath, content in dic.items(): + path = target_dir.join(relpath) + if not path.dirpath().exists(): + path.dirpath().ensure(dir=1) + path.write_binary(content) + @pytest.fixture def data(request): @@ -225,6 +270,16 @@ def __init__(self, testprocess, init_time): self.testprocess = testprocess self.init_time = init_time + def log(self, *args): + print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args) + + def add_configured(self, account): + """ add an already configured account. """ + assert account.is_configured() + self._account2state[account] = self.CONFIGURED + self.log("added already configured account", account, account.get_config("addr")) + return + def start_configure(self, account, reconfigure=False): """ add an account and start its configure process. """ class PendingTracker: @@ -235,7 +290,7 @@ def ac_configure_completed(this, success): account.add_account_plugin(PendingTracker(), name="pending_tracker") self._account2state[account] = self.CONFIGURING account.configure(reconfigure=reconfigure) - print("started configure on pending", account) + self.log("started configure on", account) def wait_one_configured(self, account): """ wait until this account has successfully configured. """ @@ -328,6 +383,9 @@ def __init__(self, request, testprocess, tmpdir, data) -> None: self.set_logging_default(False) request.addfinalizer(self.finalize) + def log(self, *args): + print("[acfactory]", "{:.3f}".format(time.time() - self.init_time), *args) + def finalize(self): while self._finalizers: fin = self._finalizers.pop() @@ -359,9 +417,19 @@ def get_next_liveconfig(self): assert "addr" in configdict and "mail_pw" in configdict return configdict + def _get_cached_account(self, addr): + if addr in self.testprocess._addr2files: + return self._getaccount(addr) + def get_unconfigured_account(self): + return self._getaccount() + + def _getaccount(self, try_cache_addr=None): logid = "ac{}".format(len(self._accounts) + 1) - path = self.tmpdir.join(logid) + # we need to use fixed database basename for maybe_cache_* functions to work + path = self.tmpdir.mkdir(logid).join("dc.db") + if try_cache_addr: + self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path) ac = Account(path.strpath, logging=self._logging) ac._logid = logid # later instantiated FFIEventLogger needs this ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac)) @@ -394,7 +462,7 @@ def _preconfigure_key(self, account, addr): def get_pseudo_configured_account(self): # do a pseudo-configured account ac = self.get_unconfigured_account() - acname = os.path.basename(ac.db_path) + acname = ac._logid addr = "{}@offline.org".format(acname) ac.update_config(dict( addr=addr, displayname=acname, mail_pw="123", @@ -405,7 +473,7 @@ def get_pseudo_configured_account(self): self._acsetup.init_logging(ac) return ac - def new_online_configuring_account(self, cloned_from=None, **kwargs): + def new_online_configuring_account(self, cloned_from=None, cache=False, **kwargs): if cloned_from is None: configdict = self.get_next_liveconfig() else: @@ -415,6 +483,12 @@ def new_online_configuring_account(self, cloned_from=None, **kwargs): mail_pw=cloned_from.get_config("mail_pw"), ) configdict.update(kwargs) + ac = self._get_cached_account(addr=configdict["addr"]) if cache else None + if ac is not None: + # make sure we consume a preconfig key, as if we had created a fresh account + self._preconfigured_keys.pop(0) + self._acsetup.add_configured(ac) + return ac ac = self.prepare_account_from_liveconfig(configdict) self._acsetup.start_configure(ac) return ac @@ -439,9 +513,11 @@ def bring_accounts_online(self): print("all accounts online") def get_online_accounts(self, num): - # to reduce number of log events logging starts after accounts can receive - accounts = [self.new_online_configuring_account() for i in range(num)] + accounts = [self.new_online_configuring_account(cache=True) for i in range(num)] self.bring_accounts_online() + # we cache fully configured and started accounts + for acc in accounts: + self.testprocess.cache_maybe_store_configured_db_files(acc) return accounts def run_bot_process(self, module, ffi=True): diff --git a/python/tests/test_4_lowlevel.py b/python/tests/test_4_lowlevel.py index 9983c23ba5..30749d8928 100644 --- a/python/tests/test_4_lowlevel.py +++ b/python/tests/test_4_lowlevel.py @@ -1,4 +1,5 @@ -from __future__ import print_function + +import os from queue import Queue from deltachat import capi, cutil, const @@ -46,6 +47,17 @@ def test_two_accounts_one_waited_all_started(self, monkeypatch, acfactory, testp assert pc._account2state[ac1] == pc.IDLEREADY assert pc._account2state[ac2] == pc.IDLEREADY + def test_store_and_retrieve_configured_account_cache(self, acfactory, tmpdir): + ac1 = acfactory.get_pseudo_configured_account() + holder = acfactory._acsetup.testprocess + assert holder.cache_maybe_store_configured_db_files(ac1) + assert not holder.cache_maybe_store_configured_db_files(ac1) + acdir = tmpdir.mkdir("newaccount") + addr = ac1.get_config("addr") + target_db_path = acdir.join("db").strpath + assert holder.cache_maybe_retrieve_configured_db_files(addr, target_db_path) + assert len(os.listdir(acdir)) >= 2 + def test_liveconfig_caching(acfactory, monkeypatch): prod = [