Commit e7d1c736 authored by Tobias Winchen's avatar Tobias Winchen
Browse files

Merge branch 'HDf5_File_Close_and_Reopen' into 'dev'

Hdf5 file close and reopen

See merge request mpifr-bdg/mpikat!122
parents c6cbe06c 91555afa
......@@ -85,9 +85,10 @@ class EDDHDFFileWriter(object):
_log.debug('Using file: {}'.format(ofile))
self.__filename = ofile
self.__mode = mode
self.__chunksize = chunksize
self._file = h5py.File(self.__filename, mode)
self._file = h5py.File(self.__filename, self.__mode)
self._file.attrs['FORMAT_VERSION'] = self._file_format_version
now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%f")
......@@ -98,11 +99,18 @@ class EDDHDFFileWriter(object):
history[-1] = a
self.__subscan_id = 0
self.__subscan = None
self.__items = []
self._lock = Lock()
def __del__(self):
try:
self._file.close()
except:
pass
@property
def filename(self):
return self.__filename
......@@ -125,7 +133,12 @@ class EDDHDFFileWriter(object):
scanid = "scan/{:03}".format(scannum)
_log.debug('Starting new subscan: {}'.format(scanid))
self.__subscan = self._file.create_group(scanid)
self._file.create_group(scanid)
def getCurrentScanId(self):
return "scan/{:03}".format(len(self._file['scan'].keys()) - 1)
def addData(self, section, data, attributes={}):
"""
......@@ -144,22 +157,23 @@ class EDDHDFFileWriter(object):
Format management
"""
with self._lock:
if section not in self.__subscan:
_log.debug('Creating new section {} for subscan: {}'.format(section, self.__subscan.name))
self.__subscan.create_group(section)
__subscan = self._file.get(self.getCurrentScanId())
if section not in __subscan:
_log.debug('Creating new section {} for subscan: {}'.format(section, __subscan.name))
__subscan.create_group(section)
# columns = gated_spectrometer_format
for k, c in data.items():
if self.__chunksize == 'auto':
self.__subscan[section].create_dataset(k, dtype=c.dtype, shape=(0,) + c.shape, maxshape=(None,)+ c.shape, chunks=True)
__subscan[section].create_dataset(k, dtype=c.dtype, shape=(0,) + c.shape, maxshape=(None,)+ c.shape, chunks=True)
else:
self.__subscan[section].create_dataset(k, dtype=c.dtype, shape=(0,) + c.shape, maxshape=(None,)+ c.shape, chunks=(self.__chunksize, )+ c.shape, )
__subscan[section].create_dataset(k, dtype=c.dtype, shape=(0,) + c.shape, maxshape=(None,)+ c.shape, chunks=(self.__chunksize, )+ c.shape, )
self.__items = set(data.keys())
if set(data.keys()) != self.__items:
_log.warning("Missing keys in dataset: {} - Ignoring dataset!".format(",".join(self.__items.difference(data.keys()))))
return
for did, dataset in self.__subscan[section].items():
for did, dataset in __subscan[section].items():
shape = list(dataset.shape)
shape[0] += 1
_log.debug('Resizing {}: {} -> {}'.format(dataset.name, dataset.shape, tuple(shape)))
......@@ -167,9 +181,18 @@ class EDDHDFFileWriter(object):
dataset[-1] = data[did]
for key, value in attributes.items():
if key not in self.__subscan[section].attrs.keys():
if key not in __subscan[section].attrs.keys():
_log.debug("Adding attribute: {} = {} to section {}".format(key, value, section))
self.__subscan[section].attrs[key] = value
__subscan[section].attrs[key] = value
def open(self, mode='None'):
"""
Re-Opens the HDF file. Use the previous mode if no other mode is given..
"""
_log.debug('Opening : {}'.format(self.filename))
self._file.close()
self._file = h5py.File(self.__filename, self.__mode)
def close(self):
"""
......
......@@ -427,6 +427,9 @@ class EDDHDF5WriterPipeline(EDDPipeline):
self._output_file = EDDHDFFileWriter(path=path, file_id_no=file_id)
self._current_file.set_value(self._output_file.filename)
else:
self._output_file.open()
if ("override_newsubscan" in config and config["override_newsubscan"]):
......@@ -450,6 +453,8 @@ class EDDHDF5WriterPipeline(EDDPipeline):
def measurement_stop(self):
_log.info("Stopping writing data to file")
self._output_file.flush()
self._output_file.close()
# ToDo: There probably should be an output queue so that time ordering
# is not becoming an issue
......
......@@ -82,8 +82,26 @@ class TestEDDHdfFileWriter(unittest.TestCase):
self.assertTrue('FORMAT_VERSION' in f._file.attrs.keys())
f.close()
def test_file_reopen(self):
fn = '/tmp/{}'.format(uuid.uuid4())
f = EDDHDFFileWriter(fn)
f.newSubscan()
self.addCleanup(os.remove, f.filename)
f.addData('testdata1', {'foo': np.zeros(5)})
f.close()
with h5py.File(f.filename, "r") as infile:
self.assertTrue("scan/000/testdata1" in infile)
with self.assertRaises(Exception) as context:
f.addData('testdata2', np.zeros(5))
f.open()
f.addData('testdata2', {'foo': np.zeros(5)})
f.close()
with h5py.File(f.filename, "r") as infile:
self.assertTrue("scan/000/testdata1" in infile)
self.assertTrue("scan/000/testdata2" in infile)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG, format='%(name)s %(levelname)s %(message)s')
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment