fix read and write of NaN to hdf5 #1180
Conversation
qcodes/data/hdf5_format.py
Outdated
| @@ -79,6 +79,7 @@ def read(self, data_set, location=None): | |||
| data_set._h5_base_group['Data Arrays'].keys()): | |||
| # Decoding string is needed because of h5py/issues/379 | |||
| name = array_id # will be overwritten if not in file | |||
| print('read array_id %s' % array_id) | |||
There was a problem hiding this comment.
What is this print statement needed for?
qcodes/data/hdf5_format.py
Outdated
| @@ -99,6 +100,9 @@ def read(self, data_set, location=None): | |||
| # set_arrays = () | |||
| vals = dat_arr.value[:, 0] | |||
| if 'shape' in dat_arr.attrs.keys(): | |||
| # extend with NaN if needed | |||
| esize = np.prod(dat_arr.attrs['shape']) | |||
| vals=np.append(vals, [np.nan]*(esize-vals.size)) | |||
There was a problem hiding this comment.
please add spaces around = for readability (around * and - as well?)
qcodes/data/hdf5_format.py
Outdated
| # get latest NaN element | ||
| new_dlen=(~np.isnan(x)).nonzero()[0][-1]+1 | ||
| except: | ||
| new_dlen=old_dlen |
There was a problem hiding this comment.
please add spaces around = signs
qcodes/data/hdf5_format.py
Outdated
| new_dlen = len(x[~np.isnan(x)]) | ||
| try: | ||
| # get latest NaN element | ||
| new_dlen=(~np.isnan(x)).nonzero()[0][-1]+1 |
There was a problem hiding this comment.
could you "unwrap" this line of code so that it is clear what is going on? actually, what is going on? :) also, is it possible to avoid the use of try-except (if you are not "excepting" a very specific exception, this pattern can easily hide possible bugs in the code inside the try block)?
| @@ -99,6 +100,9 @@ def read(self, data_set, location=None): | |||
| # set_arrays = () | |||
| vals = dat_arr.value[:, 0] | |||
| if 'shape' in dat_arr.attrs.keys(): | |||
| # extend with NaN if needed | |||
There was a problem hiding this comment.
please add a test for this.
| @@ -207,7 +211,12 @@ def write(self, data_set, io_manager=None, location=None, | |||
| datasetshape = dset.shape | |||
| old_dlen = datasetshape[0] | |||
| x = data_set.arrays[array_id] | |||
| new_dlen = len(x[~np.isnan(x)]) | |||
| try: | |||
| # get latest NaN element | |||
There was a problem hiding this comment.
please add a test for this.
|
@astafan8 Fixed the issues. I will not write tests (formatter is for the 'old' dataset) |
There was a problem hiding this comment.
@peendebak ok, I understand. But could you at least please:
- fix the broken tests (see Travis CI build failed)?
- add the code from the issue hdf5format breaks when reading incomplete datasets #1175 description as a test?
Codecov Report
@@ Coverage Diff @@
## master #1180 +/- ##
==========================================
+ Coverage 79.66% 79.67% +0.01%
==========================================
Files 47 47
Lines 6662 6667 +5
==========================================
+ Hits 5307 5312 +5
Misses 1355 1355 |
|
@astafan8 Fixed the tests. |
Merge: 1291672 a46fed2 Author: Mikhail Astafev <[email protected]> Merge pull request #1180 from VandersypenQutech/fix/hdf5nan
Fixes #1175
This solves writing datasets containing NaN elements to hdf5 format.
@dpfranke Can you check the PR and clean up where needed?
@jenshnielsen @WilliamHPNielsen