Commit b61ff2d1 authored by Jait Dixit's avatar Jait Dixit
Browse files

Update serialization for all classes

- Use attrs for storing metadata
- Remove pickle usage from Field and PowerSpace
parent 51663847
from __future__ import division from __future__ import division
import pickle
import numpy as np import numpy as np
from keepers import Versionable from keepers import Versionable
...@@ -893,14 +892,13 @@ class Field(Loggable, Versionable, object): ...@@ -893,14 +892,13 @@ class Field(Loggable, Versionable, object):
# ---Serialization--- # ---Serialization---
def _to_hdf5(self, hdf5_group): def _to_hdf5(self, hdf5_group):
# pickling for nested tuples # metadata
hdf5_group['field_type_axes'] = pickle.dumps(self.field_type_axes) hdf5_group.attrs['dtype'] = self.dtype.name
hdf5_group['domain_axes'] = pickle.dumps(self.domain_axes) hdf5_group.attrs['distribution_strategy'] = self.distribution_strategy
hdf5_group.attrs['field_type_axes'] = str(self.field_type_axes)
hdf5_group['dtype'] = self.dtype.name hdf5_group.attrs['domain_axes'] = str(self.domain_axes)
hdf5_group['distribution_strategy'] = self.distribution_strategy hdf5_group.attrs['num_domain'] = len(self.domain)
hdf5_group['num_domain'] = len(self.domain) hdf5_group.attrs['num_ft'] = len(self.field_type)
hdf5_group['num_ft'] = len(self.field_type)
ret_dict = { ret_dict = {
'val' : self.val 'val' : self.val
...@@ -923,20 +921,20 @@ class Field(Loggable, Versionable, object): ...@@ -923,20 +921,20 @@ class Field(Loggable, Versionable, object):
new_field.__class__ = cls new_field.__class__ = cls
# set values # set values
temp_domain = [] temp_domain = []
for i in range(hdf5_group['num_domain'][()]): for i in range(hdf5_group.attrs['num_domain']):
temp_domain.append(loopback_get('s_' + str(i))) temp_domain.append(loopback_get('s_' + str(i)))
new_field.domain = tuple(temp_domain) new_field.domain = tuple(temp_domain)
temp_ft = [] temp_ft = []
for i in range(hdf5_group['num_ft'][()]): for i in range(hdf5_group.attrs['num_ft']):
temp_domain.append(loopback_get('ft_' + str(i))) temp_domain.append(loopback_get('ft_' + str(i)))
new_field.field_type = tuple(temp_ft) new_field.field_type = tuple(temp_ft)
new_field.domain_axes = pickle.loads(hdf5_group['domain_axes'][()]) exec('new_field.domain_axes = ' + hdf5_group.attrs['domain_axes'])
new_field.field_type_axes = pickle.loads(hdf5_group['field_type_axes'][()]) exec('new_field.field_type_axes = ' + hdf5_group.attrs['field_type_axes'])
new_field._val = loopback_get('val') new_field._val = loopback_get('val')
new_field.dtype = np.dtype(hdf5_group['dtype'][()]) new_field.dtype = np.dtype(hdf5_group.attrs['dtype'])
new_field.distribution_strategy = hdf5_group['distribution_strategy'][()] new_field.distribution_strategy = hdf5_group.attrs['distribution_strategy']
return new_field return new_field
......
from __future__ import division from __future__ import division
import pickle
import itertools import itertools
import numpy as np import numpy as np
...@@ -217,7 +215,8 @@ class GLSpace(Versionable, Space): ...@@ -217,7 +215,8 @@ class GLSpace(Versionable, Space):
def _to_hdf5(self, hdf5_group): def _to_hdf5(self, hdf5_group):
hdf5_group['nlat'] = self.nlat hdf5_group['nlat'] = self.nlat
hdf5_group['nlon'] = self.nlon hdf5_group['nlon'] = self.nlon
hdf5_group['dtype'] = self.dtype.name # metadata
hdf5_group.attrs['dtype'] = self.dtype.name
return None return None
...@@ -226,7 +225,7 @@ class GLSpace(Versionable, Space): ...@@ -226,7 +225,7 @@ class GLSpace(Versionable, Space):
result = cls( result = cls(
nlat=hdf5_group['nlat'][()], nlat=hdf5_group['nlat'][()],
nlon=hdf5_group['nlon'][()], nlon=hdf5_group['nlon'][()],
dtype=np.dtype(hdf5_group['dtype'][()]) dtype=np.dtype(hdf5_group.attrs['dtype'])
) )
return result return result
...@@ -209,13 +209,16 @@ class HPSpace(Versionable, Space): ...@@ -209,13 +209,16 @@ class HPSpace(Versionable, Space):
def _to_hdf5(self, hdf5_group): def _to_hdf5(self, hdf5_group):
hdf5_group['nside'] = self.nside hdf5_group['nside'] = self.nside
hdf5_group['dtype'] = self.dtype.name # metadata
hdf5_group.attrs['dtype'] = self.dtype.name
return None return None
@classmethod @classmethod
def _from_hdf5(cls, hdf5_group, loopback_get): def _from_hdf5(cls, hdf5_group, loopback_get):
result = cls( result = cls(
nside=hdf5_group['nside'][()], nside=hdf5_group['nside'][()],
dtype=np.dtype(hdf5_group['dtype'][()]) dtype=np.dtype(hdf5_group.attrs['dtype'])
) )
return result return result
...@@ -187,13 +187,15 @@ class LMSpace(Versionable, Space): ...@@ -187,13 +187,15 @@ class LMSpace(Versionable, Space):
def _to_hdf5(self, hdf5_group): def _to_hdf5(self, hdf5_group):
hdf5_group['lmax'] = self.lmax hdf5_group['lmax'] = self.lmax
hdf5_group['dtype'] = self.dtype.name # metadata
hdf5_group.attrs['dtype'] = self.dtype.name
return None return None
@classmethod @classmethod
def _from_hdf5(cls, hdf5_group, loopback_get): def _from_hdf5(cls, hdf5_group, loopback_get):
result = cls( result = cls(
lmax=hdf5_group['lmax'][()], lmax=hdf5_group['lmax'][()],
dtype=np.dtype(hdf5_group['dtype'][()]) dtype=np.dtype(hdf5_group.attrs['dtype'])
) )
return result return result
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import pickle
import numpy as np import numpy as np
from keepers import Versionable from keepers import Versionable
...@@ -161,13 +160,14 @@ class PowerSpace(Versionable, Space): ...@@ -161,13 +160,14 @@ class PowerSpace(Versionable, Space):
# ---Serialization--- # ---Serialization---
def _to_hdf5(self, hdf5_group): def _to_hdf5(self, hdf5_group):
hdf5_group['log'] = self.log
hdf5_group['nbin'] = pickle.dumps(self.nbin)
hdf5_group['binbounds'] = pickle.dumps(self.binbounds)
hdf5_group['kindex'] = self.kindex hdf5_group['kindex'] = self.kindex
hdf5_group['rho'] = self.rho hdf5_group['rho'] = self.rho
hdf5_group['pundex'] = self.pundex hdf5_group['pundex'] = self.pundex
hdf5_group['dtype'] = self.dtype.name # metadata
hdf5_group.attrs['dtype'] = self.dtype.name
hdf5_group.attrs['log'] = self.log
hdf5_group.attrs['nbin'] = str(self.nbin)
hdf5_group.attrs['binbounds'] = str(self.binbounds)
return { return {
'harmonic_domain': self.harmonic_domain, 'harmonic_domain': self.harmonic_domain,
...@@ -182,11 +182,11 @@ class PowerSpace(Versionable, Space): ...@@ -182,11 +182,11 @@ class PowerSpace(Versionable, Space):
# reset class # reset class
new_ps.__class__ = cls new_ps.__class__ = cls
# set all values # set all values
new_ps.dtype = np.dtype(hdf5_group['dtype'][()]) new_ps.dtype = np.dtype(hdf5_group.attrs['dtype'])
new_ps._harmonic_domain = loopback_get('harmonic_domain') new_ps._harmonic_domain = loopback_get('harmonic_domain')
new_ps._log = hdf5_group['log'][()] new_ps._log = hdf5_group.attrs['log']
new_ps._nbin = pickle.loads(hdf5_group['nbin'][()]) exec('new_ps._nbin = ' + hdf5_group.attrs['nbin'])
new_ps._binbounds = pickle.loads(hdf5_group['binbounds'][()]) exec('new_ps._binbounds = ' + hdf5_group.attrs['binbounds'])
new_ps._pindex = loopback_get('pindex') new_ps._pindex = loopback_get('pindex')
new_ps._kindex = hdf5_group['kindex'][:] new_ps._kindex = hdf5_group['kindex'][:]
......
...@@ -328,8 +328,9 @@ class RGSpace(Versionable, Space): ...@@ -328,8 +328,9 @@ class RGSpace(Versionable, Space):
hdf5_group['shape'] = self.shape hdf5_group['shape'] = self.shape
hdf5_group['zerocenter'] = self.zerocenter hdf5_group['zerocenter'] = self.zerocenter
hdf5_group['distances'] = self.distances hdf5_group['distances'] = self.distances
hdf5_group['harmonic'] = self.harmonic # metadata
hdf5_group['dtype'] = self.dtype.name hdf5_group.attrs['harmonic'] = self.harmonic
hdf5_group.attrs['dtype'] = self.dtype.name
return None return None
...@@ -339,7 +340,7 @@ class RGSpace(Versionable, Space): ...@@ -339,7 +340,7 @@ class RGSpace(Versionable, Space):
shape=hdf5_group['shape'][:], shape=hdf5_group['shape'][:],
zerocenter=hdf5_group['zerocenter'][:], zerocenter=hdf5_group['zerocenter'][:],
distances=hdf5_group['distances'][:], distances=hdf5_group['distances'][:],
harmonic=hdf5_group['harmonic'][()], harmonic=hdf5_group.attrs['harmonic'],
dtype=np.dtype(hdf5_group['dtype'][()]) dtype=np.dtype(hdf5_group.attrs['dtype'])
) )
return result return result
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment