Commit b3cb661c authored by theos's avatar theos
Browse files

Moved d2o into dedicated subpackage.

parent 320ab0c7
...@@ -37,7 +37,7 @@ from nifty_core import space,\ ...@@ -37,7 +37,7 @@ from nifty_core import space,\
point_space,\ point_space,\
field field
from nifty_mpi_data import distributed_data_object, d2o_librarian from d2o import distributed_data_object, d2o_librarian
from nifty_random import random from nifty_random import random
from nifty_simple_math import * from nifty_simple_math import *
from nifty_utilities import * from nifty_utilities import *
......
# -*- coding: utf-8 -*-
from __future__ import division
from distributed_data_object import distributed_data_object
from d2o_librarian import d2o_librarian
from strategies import STRATEGIES
\ No newline at end of file
# -*- coding: utf-8 -*-
import numpy as np
class d2o_iter(object):
def __init__(self, d2o):
self.d2o = d2o
self.i = 0
self.n = np.prod(self.d2o.shape)
self.initialize_current_local_data()
def __iter__(self):
return self
def next(self):
if self.n == 0:
raise StopIteration()
self.update_current_local_data()
if self.i < self.n:
i = self.i
self.i += 1
return self.current_local_data[i]
else:
raise StopIteration()
def initialize_current_local_data(self):
raise NotImplementedError
def update_current_local_data(self):
raise NotImplementedError
class d2o_not_iter(d2o_iter):
def initialize_current_local_data(self):
self.current_local_data = self.d2o.data.flatten()
def update_current_local_data(self):
pass
class d2o_slicing_iter(d2o_iter):
def __init__(self, d2o):
self.d2o = d2o
self.i = 0
self.n = np.prod(self.d2o.shape)
self.local_dim_offset_list = \
self.d2o.distributor.all_local_slices[:, 4]
self.active_node = None
self.initialize_current_local_data()
def initialize_current_local_data(self):
self.update_current_local_data()
def update_current_local_data(self):
new_active_node = np.searchsorted(self.local_dim_offset_list,
self.i,
side='right')-1
# new_active_node = min(new_active_node, self.d2o.comm.size-1)
if self.active_node != new_active_node:
self.active_node = new_active_node
self.current_local_data = self.d2o.comm.bcast(
self.d2o.get_local_data().flatten(),
root=self.active_node)
# -*- coding: utf-8 -*-
from weakref import WeakValueDictionary as weakdict
class _d2o_librarian(object):
def __init__(self):
self.library = weakdict()
self.counter = 0
def register(self, d2o):
self.counter += 1
self.library[self.counter] = d2o
return self.counter
def __getitem__(self, key):
return self.library[key]
d2o_librarian = _d2o_librarian()
This diff is collapsed.
# -*- coding: utf-8 -*-
import numpy as np
from nifty.keepers import global_configuration as gc,\
global_dependency_injector as gdi
MPI = gdi[gc['mpi_module']]
class _dtype_converter(object):
"""
NIFTY class for dtype conversion between python/numpy dtypes and MPI
dtypes.
"""
def __init__(self):
pre_dict = [
# [, MPI_CHAR],
# [, MPI_SIGNED_CHAR],
# [, MPI_UNSIGNED_CHAR],
[np.dtype('bool'), MPI.BYTE],
[np.dtype('int16'), MPI.SHORT],
[np.dtype('uint16'), MPI.UNSIGNED_SHORT],
[np.dtype('uint32'), MPI.UNSIGNED_INT],
[np.dtype('int32'), MPI.INT],
[np.dtype('int'), MPI.LONG],
[np.dtype(np.long), MPI.LONG],
[np.dtype('int64'), MPI.LONG_LONG],
[np.dtype('longlong'), MPI.LONG],
[np.dtype('uint'), MPI.UNSIGNED_LONG],
[np.dtype('uint64'), MPI.UNSIGNED_LONG_LONG],
[np.dtype('ulonglong'), MPI.UNSIGNED_LONG_LONG],
[np.dtype('float32'), MPI.FLOAT],
[np.dtype('float64'), MPI.DOUBLE],
[np.dtype('float128'), MPI.LONG_DOUBLE],
[np.dtype('complex64'), MPI.COMPLEX],
[np.dtype('complex128'), MPI.DOUBLE_COMPLEX]]
to_mpi_pre_dict = np.array(pre_dict)
to_mpi_pre_dict[:, 0] = map(self.dictionize_np, to_mpi_pre_dict[:, 0])
self._to_mpi_dict = dict(to_mpi_pre_dict)
to_np_pre_dict = np.array(pre_dict)[:, ::-1]
to_np_pre_dict[:, 0] = map(self.dictionize_mpi, to_np_pre_dict[:, 0])
self._to_np_dict = dict(to_np_pre_dict)
def dictionize_np(self, x):
dic = x.type.__dict__.items()
if x.type is np.float:
dic[24] = 0
dic[29] = 0
dic[37] = 0
return frozenset(dic)
def dictionize_mpi(self, x):
return x.name
def to_mpi(self, dtype):
return self._to_mpi_dict[self.dictionize_np(dtype)]
def to_np(self, dtype):
return self._to_np_dict[self.dictionize_mpi(dtype)]
def known_mpi_Q(self, dtype):
return (self.dictionize_mpi(dtype) in self._to_np_dict)
def known_np_Q(self, dtype):
return (self.dictionize_np(np.dtype(dtype)) in self._to_mpi_dict)
dtype_converter = _dtype_converter()
# -*- coding: utf-8 -*-
from nifty.keepers import global_dependency_injector as gdi
pyfftw = gdi.get('pyfftw')
_maybe_fftw = ['fftw'] if ('pyfftw' in gdi) else []
STRATEGIES = {
'all': ['not', 'equal', 'freeform'] + _maybe_fftw,
'global': ['not', 'equal'] + _maybe_fftw,
'local': ['freeform'],
'slicing': ['equal', 'freeform'] + _maybe_fftw,
'not': ['not'],
'hdf5': ['equal'] + _maybe_fftw,
}
...@@ -50,8 +50,7 @@ from nifty.nifty_paradict import lm_space_paradict,\ ...@@ -50,8 +50,7 @@ from nifty.nifty_paradict import lm_space_paradict,\
hp_space_paradict hp_space_paradict
from nifty.nifty_power_indices import lm_power_indices from nifty.nifty_power_indices import lm_power_indices
from nifty.nifty_mpi_data import distributed_data_object from nifty.d2o import STRATEGIES as DISTRIBUTION_STRATEGIES
from nifty.nifty_mpi_data import STRATEGIES as DISTRIBUTION_STRATEGIES
from nifty.nifty_random import random from nifty.nifty_random import random
......
...@@ -152,8 +152,8 @@ from keepers import about,\ ...@@ -152,8 +152,8 @@ from keepers import about,\
global_dependency_injector as gdi global_dependency_injector as gdi
from nifty_random import random from nifty_random import random
from nifty.nifty_mpi_data import distributed_data_object,\ from nifty.d2o import distributed_data_object,\
STRATEGIES as DISTRIBUTION_STRATEGIES STRATEGIES as DISTRIBUTION_STRATEGIES
import nifty.nifty_utilities as utilities import nifty.nifty_utilities as utilities
......
...@@ -4,8 +4,8 @@ import numpy as np ...@@ -4,8 +4,8 @@ import numpy as np
from nifty.keepers import about,\ from nifty.keepers import about,\
global_configuration as gc,\ global_configuration as gc,\
global_dependency_injector as gdi global_dependency_injector as gdi
from nifty.nifty_mpi_data import distributed_data_object,\ from nifty.d2o import distributed_data_object,\
STRATEGIES as DISTRIBUTION_STRATEGIES STRATEGIES as DISTRIBUTION_STRATEGIES
MPI = gdi[gc['mpi_module']] MPI = gdi[gc['mpi_module']]
hp = gdi.get('healpy') hp = gdi.get('healpy')
......
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
import numpy as np import numpy as np
from keepers import about from keepers import about
from nifty_mpi_data import distributed_data_object from nifty.d2o import distributed_data_object
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
......
...@@ -9,8 +9,8 @@ from nifty.keepers import about,\ ...@@ -9,8 +9,8 @@ from nifty.keepers import about,\
global_dependency_injector as gdi,\ global_dependency_injector as gdi,\
global_configuration as gc global_configuration as gc
from nifty.nifty_mpi_data import distributed_data_object,\ from nifty.d2o import distributed_data_object,\
STRATEGIES STRATEGIES
from nifty.nifty_core import point_space,\ from nifty.nifty_core import point_space,\
field field
from nifty.rg import rg_space from nifty.rg import rg_space
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import numpy as np import numpy as np
from nifty.nifty_mpi_data import distributed_data_object from nifty.d2o import distributed_data_object
from nifty.keepers import about,\ from nifty.keepers import global_dependency_injector as gdi
global_dependency_injector as gdi
pyfftw = gdi.get('pyfftw') pyfftw = gdi.get('pyfftw')
gfft = gdi.get('gfft') gfft = gdi.get('gfft')
......
...@@ -47,8 +47,8 @@ import nifty_fft ...@@ -47,8 +47,8 @@ import nifty_fft
from nifty.keepers import about,\ from nifty.keepers import about,\
global_dependency_injector as gdi,\ global_dependency_injector as gdi,\
global_configuration as gc global_configuration as gc
from nifty.nifty_mpi_data import distributed_data_object from nifty.d2o import distributed_data_object,\
from nifty.nifty_mpi_data import STRATEGIES as DISTRIBUTION_STRATEGIES STRATEGIES as DISTRIBUTION_STRATEGIES
from nifty.nifty_paradict import rg_space_paradict from nifty.nifty_paradict import rg_space_paradict
from nifty.nifty_power_indices import rg_power_indices from nifty.nifty_power_indices import rg_power_indices
from nifty.nifty_random import random from nifty.nifty_random import random
......
...@@ -14,8 +14,8 @@ import warnings ...@@ -14,8 +14,8 @@ import warnings
import tempfile import tempfile
import nifty import nifty
from nifty.nifty_mpi_data import distributed_data_object,\ from nifty.d2o import distributed_data_object,\
STRATEGIES STRATEGIES
FOUND = {} FOUND = {}
try: try:
......
...@@ -11,9 +11,9 @@ import unittest ...@@ -11,9 +11,9 @@ import unittest
from nifty.nifty_utilities import hermitianize,\ from nifty.nifty_utilities import hermitianize,\
_hermitianize_inverter _hermitianize_inverter
from nifty.nifty_mpi_data import distributed_data_object,\ from nifty.d2o import distributed_data_object,\
STRATEGIES STRATEGIES
############################################################################### ###############################################################################
def custom_name_func(testcase_func, param_num, param): def custom_name_func(testcase_func, param_num, param):
...@@ -43,12 +43,12 @@ flipped_data = np.array([[-10, -8, -7, 2, 10, 9], ...@@ -43,12 +43,12 @@ flipped_data = np.array([[-10, -8, -7, 2, 10, 9],
class Test_hermitianize_inverter(unittest.TestCase): class Test_hermitianize_inverter(unittest.TestCase):
def test_with_ndarray(self): def test_with_ndarray(self):
assert_equal(_hermitianize_inverter(test_data), flipped_data) assert_equal(_hermitianize_inverter(test_data), flipped_data)
@parameterized.expand(STRATEGIES['global'], @parameterized.expand(STRATEGIES['global'],
testcase_func_name=custom_name_func) testcase_func_name=custom_name_func)
def test_with_d2o(self, distribution_strategy): def test_with_d2o(self, distribution_strategy):
d = distributed_data_object( d = distributed_data_object(
test_data, test_data,
distribution_strategy=distribution_strategy) distribution_strategy=distribution_strategy)
assert_equal(_hermitianize_inverter(d).get_full_data(), flipped_data) assert_equal(_hermitianize_inverter(d).get_full_data(), flipped_data)
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment