Merge branch 'master' into docs/dataexport_notebook

.travis.yml

@@ -41,12 +41,14 @@ script:
     - |
       pip uninstall -y qcodes
       pip install -e .
-      cd qcodes/tests/dataset/legacy_DB_generation
+      cd ..
+      git clone https://github.com/QCoDeS/qcodes_generate_test_db.git
+      cd qcodes_generate_test_db
       python generate_version_0.py
       python generate_version_1.py
       python generate_version_2.py
       python generate_version_3.py
-      cd ../../../..
+      cd $TRAVIS_BUILD_DIR
       pip uninstall -y qcodes
       pip install .
     - cd qcodes

azure-pipelines.yml

@@ -29,11 +29,13 @@ jobs:
           mypy qcodes
         displayName: "mypy"
       - script: |
-          cd qcodes
-          python tests\dataset\legacy_DB_generation\generate_version_0.py
-          python tests\dataset\legacy_DB_generation\generate_version_1.py
-          python tests\dataset\legacy_DB_generation\generate_version_2.py
-          python tests\dataset\legacy_DB_generation\generate_version_3.py
+          cd ..
+          git clone https://github.com/QCoDeS/qcodes_generate_test_db.git
+          cd qcodes_generate_test_db
+          python generate_version_0.py
+          python generate_version_1.py
+          python generate_version_2.py
+          python generate_version_3.py
         displayName: "Generate db fixtures"
       - script: |
           cd qcodes

qcodes/dataset/param_spec.py

@@ -43,10 +43,19 @@ def __init__(self, name: str,
         inferred_from = [] if inferred_from is None else inferred_from
         depends_on = [] if depends_on is None else depends_on
 
+        if isinstance(inferred_from, str):
+            raise ValueError(f"ParamSpec {self.name} got "
+                             f"string {inferred_from} as inferred_from. "
+                             f"It needs a "
+                             f"Sequence of ParamSpecs or strings")
         self._inferred_from.extend(
             p.name if isinstance(p, ParamSpec) else p
             for p in inferred_from)
 
+        if isinstance(depends_on, str):
+            raise ValueError(f"ParamSpec {self.name} got "
+                             f"string {depends_on} as depends_on. It needs a "
+                             f"Sequence of ParamSpecs or strings")
         self._depends_on.extend(
             p.name if isinstance(p, ParamSpec) else p
             for p in depends_on)

qcodes/dataset/sqlite_base.py

@@ -606,7 +606,7 @@ def _2to3_get_paramspecs(conn: ConnectionPlus,
     # depend, then the dependent ParamSpecs and finally the rest
 
     for layout_id in list(indeps) + list(deps) + list(the_rest):
-        (name, label, unit, inferred_from) = layouts[layout_id]
+        (name, label, unit, inferred_from_str) = layouts[layout_id]
         # get the data type
         sql = f'PRAGMA TABLE_INFO("{result_table_name}")'
         c = transaction(conn, sql)
@@ -615,34 +615,23 @@ def _2to3_get_paramspecs(conn: ConnectionPlus,
                 paramtype = row['type']
                 break
 
-        # first possibility: another parameter depends on this parameter
-        if layout_id in indeps:
-            paramspec = ParamSpec(name=name, paramtype=paramtype,
-                                  label=label, unit=unit,
-                                  inferred_from=inferred_from)
-            paramspecs[layout_id] = paramspec
-
-        # second possibility: this parameter depends on another parameter
-        elif layout_id in deps:
+        inferred_from: List[str] = []
+        depends_on: List[str] = []
 
+        # this parameter depends on another parameter
+        if layout_id in deps:
             setpoints = dependencies[layout_id]
             depends_on = [paramspecs[idp].name for idp in setpoints]
 
-            paramspec = ParamSpec(name=name,
-                                  paramtype=paramtype,
-                                  label=label, unit=unit,
-                                  depends_on=depends_on,
-                                  inferred_from=inferred_from)
-            paramspecs[layout_id] = paramspec
+        if inferred_from_str != '':
+            inferred_from = inferred_from_str.split(', ')
 
-        # third possibility: no dependencies
-        else:
-            paramspec = ParamSpec(name=name,
-                                  paramtype=paramtype,
-                                  label=label, unit=unit,
-                                  depends_on=[],
-                                  inferred_from=[])
-            paramspecs[layout_id] = paramspec
+        paramspec = ParamSpec(name=name,
+                              paramtype=paramtype,
+                              label=label, unit=unit,
+                              depends_on=depends_on,
+                              inferred_from=inferred_from)
+        paramspecs[layout_id] = paramspec
 
     return paramspecs
 
@@ -720,6 +709,77 @@ def perform_db_upgrade_2_to_3(conn: ConnectionPlus) -> None:
             log.debug(f"Upgrade in transition, run number {run_id}: OK")
 
 
+@upgrader
+def perform_db_upgrade_3_to_4(conn: ConnectionPlus) -> None:
+    """
+    Perform the upgrade from version 3 to version 4. This really
+    repeats the version 3 upgrade as it originally had two bugs in
+    the inferred annotation. inferred_from was passed incorrectly
+    resulting in the parameter being marked inferred_from for each char
+    in the inferred_from variable and inferred_from was not handled
+    correctly for parameters that were neither dependencies nor dependent on
+    other parameters. Both have since been fixed so rerun the upgrade.
+    """
+
+    no_of_runs_query = "SELECT max(run_id) FROM runs"
+    no_of_runs = one(atomic_transaction(conn, no_of_runs_query), 'max(run_id)')
+    no_of_runs = no_of_runs or 0
+
+    # If one run fails, we want the whole upgrade to roll back, hence the
+    # entire upgrade is one atomic transaction
+
+    with atomic(conn) as conn:
+
+        result_tables = _2to3_get_result_tables(conn)
+        layout_ids_all = _2to3_get_layout_ids(conn)
+        indeps_all = _2to3_get_indeps(conn)
+        deps_all = _2to3_get_deps(conn)
+        layouts = _2to3_get_layouts(conn)
+        dependencies = _2to3_get_dependencies(conn)
+
+        pbar = tqdm(range(1, no_of_runs+1))
+        pbar.set_description("Upgrading database")
+
+        for run_id in pbar:
+
+            if run_id in layout_ids_all:
+
+                result_table_name = result_tables[run_id]
+                layout_ids = list(layout_ids_all[run_id])
+                if run_id in indeps_all:
+                    independents = tuple(indeps_all[run_id])
+                else:
+                    independents = ()
+                if run_id in deps_all:
+                    dependents = tuple(deps_all[run_id])
+                else:
+                    dependents = ()
+
+                paramspecs = _2to3_get_paramspecs(conn,
+                                                  layout_ids,
+                                                  layouts,
+                                                  dependencies,
+                                                  dependents,
+                                                  independents,
+                                                  result_table_name)
+
+                interdeps = InterDependencies(*paramspecs.values())
+                desc = RunDescriber(interdeps=interdeps)
+                json_str = desc.to_json()
+
+            else:
+
+                json_str = RunDescriber(InterDependencies()).to_json()
+
+            sql = f"""
+                   UPDATE runs
+                   SET run_description = ?
+                   WHERE run_id == ?
+                   """
+            cur = conn.cursor()
+            cur.execute(sql, (json_str, run_id))
+            log.debug(f"Upgrade in transition, run number {run_id}: OK")
+
 def _latest_available_version() -> int:
     """Return latest available database schema version"""
     return len(_UPGRADE_ACTIONS)