From 7ffdfd38deea4eab1678f9f0bba4b061d8049580 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:13:55 +0100 Subject: [PATCH 01/10] Add hash for database version 3 --- qcodes/tests/dataset/legacy_DB_generation/utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qcodes/tests/dataset/legacy_DB_generation/utils.py b/qcodes/tests/dataset/legacy_DB_generation/utils.py index 2a594fa6505..19510e13c32 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/utils.py +++ b/qcodes/tests/dataset/legacy_DB_generation/utils.py @@ -14,11 +14,17 @@ # # Version 2: indices are added to runs; GUID and exp_id # +# Version 3: run_description column is added to the runs table +# GIT_HASHES: Dict[int, str] = {0: '78d42620fc245a975b5a615ed5e33061baac7846', 1: '056d59627e22fa3ca7aad4c265e9897c343f79cf', - 2: '5202255924542dad6841dfe3d941a7f80c43956c'} + 2: '5202255924542dad6841dfe3d941a7f80c43956c', + # NOTE: commit hash associated with version 3 + # will need to be updated once we move to + # version 4 + 3: 'a5d8edae05898df0cbfb5d9048565df91efea692'} gitrepopath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-5]) repo = Repo(gitrepopath) From 4046ab89d2325c254b1e5cf7c5ab3808c008ef99 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:14:26 +0100 Subject: [PATCH 02/10] Add note about what hashes for database upgrade testing mean --- qcodes/tests/dataset/legacy_DB_generation/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qcodes/tests/dataset/legacy_DB_generation/utils.py b/qcodes/tests/dataset/legacy_DB_generation/utils.py index 19510e13c32..ba01ffa04ad 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/utils.py +++ b/qcodes/tests/dataset/legacy_DB_generation/utils.py @@ -17,6 +17,9 @@ # Version 3: run_description column is added to the runs table # +# NOTE that each hash is supposed to be representing a commit JUST before the +# "next" version is being introduced. +# GIT_HASHES: Dict[int, str] = {0: '78d42620fc245a975b5a615ed5e33061baac7846', 1: '056d59627e22fa3ca7aad4c265e9897c343f79cf', From cb22dc42d8c8312669e513aafadd3ac505f5fa58 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:15:30 +0100 Subject: [PATCH 03/10] Add script for generating databases of version 3 one for testing bad/empty run descriptions --- .../generate_version_3.py | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py new file mode 100644 index 00000000000..90fba7ac904 --- /dev/null +++ b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py @@ -0,0 +1,115 @@ +# Generate version 3 database files for qcodes' test suite to consume + +import os +import numpy as np + +# NB: it's important that we do not import anything from qcodes before we +# do the git magic (which we do below), hence the relative import here +import utils as utils + + +fixturepath = os.sep.join(os.path.realpath(__file__).split(os.sep)[:-2]) +fixturepath = os.path.join(fixturepath, 'fixtures', 'db_files') + + +def generate_empty_DB_file(): + """ + Generate the bare minimal DB file with no runs + """ + + import qcodes.dataset.sqlite_base as sqlite_base + + v3fixturepath = os.path.join(fixturepath, 'version3') + os.makedirs(v3fixturepath, exist_ok=True) + path = os.path.join(v3fixturepath, 'empty.db') + + if os.path.exists(path): + os.remove(path) + + sqlite_base.connect(path) + + +def generate_DB_file_with_some_runs_having_not_run_descriptions(): + """ + Generate a .db-file with a handful of runs some of which lack run + description or have it as empty object (based on a real case) + """ + v3fixturepath = os.path.join(fixturepath, 'version3') + os.makedirs(v3fixturepath, exist_ok=True) + path = os.path.join(v3fixturepath, 'some_runs_without_run_description.db') + + if os.path.exists(path): + os.remove(path) + + from qcodes.dataset.measurements import Measurement + from qcodes.dataset.experiment_container import Experiment + from qcodes import Parameter + from qcodes.dataset.descriptions import RunDescriber + from qcodes.dataset.dependencies import InterDependencies + + exp = Experiment(path_to_db=path, + name='experiment_1', + sample_name='no_sample_1') + conn = exp.conn + + # Now make some parameters to use in measurements + params = [] + for n in range(5): + params.append(Parameter(f'p{n}', label=f'Parameter {n}', + unit=f'unit {n}', set_cmd=None, get_cmd=None)) + + # Set up a measurement + + meas = Measurement(exp) + meas.register_parameter(params[0]) + meas.register_parameter(params[1]) + meas.register_parameter(params[2], basis=(params[0],)) + meas.register_parameter(params[3], basis=(params[1],)) + meas.register_parameter(params[4], setpoints=(params[2], params[3])) + + # Initially make 3 correct runs + + run_ids = [] + + for _ in range(3): + + with meas.run() as datasaver: + + for x in np.random.rand(10): + for y in np.random.rand(10): + z = np.random.rand() + datasaver.add_result((params[2], x), + (params[3], y), + (params[4], z)) + + run_ids.append(datasaver.run_id) + + # Formulate SQL query for adjusting run_description column + + set_run_description_sql = f""" + UPDATE runs + SET run_description = ? + WHERE run_id == ? + """ + + # Make run_description of run #2 NULL + + conn.execute(set_run_description_sql, ('', run_ids[1])) + conn.commit() # just to be sure + + # Make run_description of run #3 equivalent to an empty RunDescriber + + empty_run_description = RunDescriber(InterDependencies()).to_json() + conn.execute(set_run_description_sql, (empty_run_description, run_ids[2])) + conn.commit() # just to be sure + + conn.close() + + +if __name__ == '__main__': + + gens = (generate_empty_DB_file, + generate_DB_file_with_some_runs_having_not_run_descriptions) + + # pylint: disable=E1101 + utils.checkout_to_old_version_and_run_generators(version=2, gens=gens) From 72c30d9f691d643932dba057b93e217e0d29a539 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:18:23 +0100 Subject: [PATCH 04/10] Generate database fixtures for testing version 3 in CI --- .travis.yml | 1 + azure-pipelines.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index f81c26e992c..55c548856f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,7 @@ script: python generate_version_0.py python generate_version_1.py python generate_version_2.py + python generate_version_3.py cd ../../../.. pip uninstall -y qcodes pip install . diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f4e5b626f19..d157f96026d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -33,6 +33,7 @@ jobs: python tests\dataset\legacy_DB_generation\generate_version_0.py python tests\dataset\legacy_DB_generation\generate_version_1.py python tests\dataset\legacy_DB_generation\generate_version_2.py + python tests\dataset\legacy_DB_generation\generate_version_3.py displayName: "Generate db fixtures" - script: | cd qcodes From d6596e6c28d1f7ccd211d2bc7664028063535f4e Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:18:38 +0100 Subject: [PATCH 05/10] Generate database fixtures for testing version 1-3 in appveyor --- .appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 256d6be6a9d..4718bb7d9bb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -28,6 +28,9 @@ install: test_script: - cd qcodes - python tests\dataset\legacy_DB_generation\generate_version_0.py + - python tests\dataset\legacy_DB_generation\generate_version_1.py + - python tests\dataset\legacy_DB_generation\generate_version_2.py + - python tests\dataset\legacy_DB_generation\generate_version_3.py - cd tests - pytest - cd ..\.. From d2e26625368495b1558be6fac12cf3641a7c6d0e Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:20:00 +0100 Subject: [PATCH 06/10] Checkout version 3 for generating version 3 database files (typo) --- qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py index 90fba7ac904..1a2b4a09d86 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py +++ b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py @@ -112,4 +112,4 @@ def generate_DB_file_with_some_runs_having_not_run_descriptions(): generate_DB_file_with_some_runs_having_not_run_descriptions) # pylint: disable=E1101 - utils.checkout_to_old_version_and_run_generators(version=2, gens=gens) + utils.checkout_to_old_version_and_run_generators(version=3, gens=gens) From b965130cd8f7615a6452594edf9c3168d78493f5 Mon Sep 17 00:00:00 2001 From: WilliamHPNielsen Date: Mon, 26 Nov 2018 14:25:48 +0100 Subject: [PATCH 07/10] Add function to fix wrong descriptions --- qcodes/dataset/sqlite_base.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/qcodes/dataset/sqlite_base.py b/qcodes/dataset/sqlite_base.py index c9e9cdb0df3..7edf3ef91f3 100644 --- a/qcodes/dataset/sqlite_base.py +++ b/qcodes/dataset/sqlite_base.py @@ -2095,3 +2095,37 @@ def _both_zero(run_id: int, conn, guid_comps) -> None: log.info(f'Updating run number {run_id}...') actions[(loc == 0, ws == 0)](run_id, conn, guid_comps) + + +def _fix_wrong_run_descriptions(conn: SomeConnection, + run_ids: Sequence[int]) -> None: + """ + NB: This is a FIX function. Do not use it unless your database has been + diagnosed with the problem that this function fixes. + + Overwrite faulty run_descriptions by using information from the layouts and + dependencies tables. If a correct description is found for a run, that + run is left untouched. + + Args: + conn: The connection to the database + run_ids: The runs to (potentially) fix + """ + + log.info('[*] Fixing run descriptions...') + for run_id in run_ids: + trusted_paramspecs = get_parameters(conn, run_id) + trusted_desc = RunDescriber( + interdeps=InterDependencies(*trusted_paramspecs)) + + actual_desc_str = select_one_where(conn, "runs", + "run_description", + "run_id", run_id) + + if actual_desc_str == trusted_desc.to_json(): + log.info(f'[+] Run id: {run_id} had an OK description') + else: + log.info(f'[-] Run id: {run_id} had a broken description. ' + f'Description found: {actual_desc_str}') + update_run_description(conn, run_id, trusted_desc.to_json()) + log.info(f' Run id: {run_id} has been updated.') From 75eac06b4a935002c74a0a604155d46ebc231b3a Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:30:14 +0100 Subject: [PATCH 08/10] Generate run without parameters and make its description NULL --- .../generate_version_3.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py index 1a2b4a09d86..ab7f85a11cd 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py +++ b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py @@ -32,7 +32,13 @@ def generate_empty_DB_file(): def generate_DB_file_with_some_runs_having_not_run_descriptions(): """ Generate a .db-file with a handful of runs some of which lack run - description or have it as empty object (based on a real case) + description or have it as empty object (based on a real case). + + Generated runs: + #1: run with parameters and correct run description + #2: run with parameters but run description is NULL + #3: run with parameters but run description is empty RunDescriber + #4: run without parameters but run description is NULL """ v3fixturepath = os.path.join(fixturepath, 'version3') os.makedirs(v3fixturepath, exist_ok=True) @@ -84,6 +90,9 @@ def generate_DB_file_with_some_runs_having_not_run_descriptions(): run_ids.append(datasaver.run_id) + assert [1, 2, 3] == run_ids, 'Run ids of generated runs are not as ' \ + 'expected after generating runs #1-3' + # Formulate SQL query for adjusting run_description column set_run_description_sql = f""" @@ -103,7 +112,22 @@ def generate_DB_file_with_some_runs_having_not_run_descriptions(): conn.execute(set_run_description_sql, (empty_run_description, run_ids[2])) conn.commit() # just to be sure - conn.close() + # Set up a measurement without parameters, and create run #4 out of it + + meas_no_params = Measurement(exp) + + with meas_no_params.run() as datasaver: + pass + + run_ids.append(datasaver.run_id) + + assert [1, 2, 3, 4] == run_ids, 'Run ids of generated runs are not as ' \ + 'expected after generating run #4' + + # Make run_description of run #4 NULL + + conn.execute(set_run_description_sql, ('', run_ids[3])) + conn.commit() # just to be sure if __name__ == '__main__': From feaab042339b997fdecc0a237533bde99477fdf1 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:34:31 +0100 Subject: [PATCH 09/10] Fix setting run description to NULL (use None in python) --- .../tests/dataset/legacy_DB_generation/generate_version_3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py index ab7f85a11cd..7d67cd50628 100644 --- a/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py +++ b/qcodes/tests/dataset/legacy_DB_generation/generate_version_3.py @@ -103,7 +103,7 @@ def generate_DB_file_with_some_runs_having_not_run_descriptions(): # Make run_description of run #2 NULL - conn.execute(set_run_description_sql, ('', run_ids[1])) + conn.execute(set_run_description_sql, (None, run_ids[1])) conn.commit() # just to be sure # Make run_description of run #3 equivalent to an empty RunDescriber @@ -126,7 +126,7 @@ def generate_DB_file_with_some_runs_having_not_run_descriptions(): # Make run_description of run #4 NULL - conn.execute(set_run_description_sql, ('', run_ids[3])) + conn.execute(set_run_description_sql, (None, run_ids[3])) conn.commit() # just to be sure From 008b240e1cd39820cc25c09fa317a6de9ce6b848 Mon Sep 17 00:00:00 2001 From: Mikhail Astafev Date: Mon, 26 Nov 2018 14:46:03 +0100 Subject: [PATCH 10/10] Add test for fixing wrong run descriptions in database version 3 --- qcodes/tests/dataset/test_database_fixes.py | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 qcodes/tests/dataset/test_database_fixes.py diff --git a/qcodes/tests/dataset/test_database_fixes.py b/qcodes/tests/dataset/test_database_fixes.py new file mode 100644 index 00000000000..bea9ad33148 --- /dev/null +++ b/qcodes/tests/dataset/test_database_fixes.py @@ -0,0 +1,45 @@ +import os + +import pytest + +import qcodes +from qcodes.dataset.data_set import DataSet +from qcodes.dataset.dependencies import InterDependencies +from qcodes.dataset.descriptions import RunDescriber +from qcodes.dataset.sqlite_base import _fix_wrong_run_descriptions, \ + get_user_version +from qcodes.tests.dataset.temporary_databases import temporarily_copied_DB + +fixturepath = os.sep.join(qcodes.tests.dataset.__file__.split(os.sep)[:-1]) +fixturepath = os.path.join(fixturepath, 'fixtures') + + +def test_fix_wrong_run_descriptions(): + v3fixpath = os.path.join(fixturepath, 'db_files', 'version3') + + if not os.path.exists(v3fixpath): + pytest.skip( + "No db-file fixtures found. You can generate test db-files" + " using the scripts in the legacy_DB_generation folder") + + dbname_old = os.path.join(v3fixpath, 'some_runs_without_run_description.db') + + with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn: + + assert get_user_version(conn) == 3 + + ds1 = DataSet(conn=conn, run_id=1) + expected_description = ds1.description + + empty_description = RunDescriber(InterDependencies()) + + _fix_wrong_run_descriptions(conn, [1, 2, 3, 4]) + + ds2 = DataSet(conn=conn, run_id=2) + assert expected_description == ds2.description + + ds3 = DataSet(conn=conn, run_id=3) + assert expected_description == ds3.description + + ds4 = DataSet(conn=conn, run_id=4) + assert empty_description == ds4.description