Commit c3b65d9c authored by David Sikter's avatar David Sikter
Browse files

Fixed issues related to lazy_import in python 3.8

parent 64d27a46
Pipeline #96104 passed with stages
in 22 minutes and 38 seconds
...@@ -16,190 +16,150 @@ ...@@ -16,190 +16,150 @@
# limitations under the License. # limitations under the License.
# #
from types import ModuleType '''
Functionality for "lazy loading". A lazy loaded module is registered in sys.modules as
loaded, but is not actually run intil needed, i.e. when an attribute which requires the
module to be imported is accessed. This is useful for optimization purposes, but needs to
be used with caution.
Since the actual import of a lazy loaded module is postponend until it is actually needed,
exceptions occuring during the actual import can happen in a location where you may not
have anticipated it. Threfore, exceptions raised during the lazy import are wrapped in a
dedicated exception class, LazyImportError, to avoid potential import exceptions being
import sys import sys
from types import ModuleType
from typing import Set
from importlib._bootstrap import _ImportLockContext from importlib._bootstrap import _ImportLockContext
from six import raise_from import importlib
from importlib import reload as reload_module from nomad import config
__all__ = ['lazy_module', 'LazyModule', '_MSG']
_not_yet_imported_lazy_module_names: Set[str] = set()
'_lazy_import_error_strings', '_lazy_import_error_msgs', '_lazy_import_callables',
'_lazy_import_submodules', '__repr__'
_DICT_DELETION = ('_lazy_import_submodules',) class LazyImportError(Exception):
_MSG = ("{caller} attempted to use a functionality that requires module "
"{module}, but it couldn't be loaded. Please install {install_name} "
"and retry.")
class _LazyModule(ModuleType):
class LazyModule(ModuleType): '''
Base class for lazy modules.
def __getattribute__(self, attr): def __getattribute__(self, attr):
if attr not in ('__name__', '__class__', '__spec__'): '''
Overrides the standard method, to trigger the actual import of the lazy module
when a non-trivial attribute is accessed.
Note: when we actually import the module, we will replace this method with the
standard method, thereby reverting to standard behaviour.
if attr not in ('__name__', '__class__', '__spec__', '__repr__', '__file__'):
try: try:
# In case the attribute we are trying to access is actually another
# lazy-loaded module, just return it.
name = '%s.%s' % (self.__name__, attr) name = '%s.%s' % (self.__name__, attr)
return sys.modules[name] return sys.modules[name]
except KeyError: except KeyError:
pass pass
# No, we have to actually load the module now!
try: _actually_import(self.__name__)
return type(self)._lazy_import_callables[attr] return ModuleType.__getattribute__(self, attr) # Standard functionality
except (AttributeError, KeyError):
return super(LazyModule, self).__getattribute__(attr)
def __setattr__(self, attr, value): def __setattr__(self, attr, value):
_load_module(self) '''
return super(LazyModule, self).__setattr__(attr, value) Overrides the standard method, to trigger the actual import of the lazy module
when any attribute is set on the module.
def _clean_lazy_submodule_refs(module): Note: when we actually import the module, we will replace this method with the
module_class = type(module) standard method, thereby reverting to standard behaviour.
for entry in _DICT_DELETION: '''
try: _actually_import(self.__name__)
names = getattr(module_class, entry) return ModuleType.__setattr__(self, attr, value) # Standard functionality
except AttributeError:
for name in names: def _actually_import(module_name):
try: '''
super(LazyModule, module).__delattr__(name) Actually import the lazy module. Also make sure that all its parent modules are
except AttributeError: imported if needed - and they need to be imported in the right order.
pass '''
def _clean_lazymodule(module):
module_class = type(module)
module_class.__getattribute__ = ModuleType.__getattribute__
module_class.__setattr__ = ModuleType.__setattr__
class_attrs = {}
for attr in _CLS_ATTRS:
class_attrs[attr] = getattr(module_class, attr)
delattr(module_class, attr)
except AttributeError:
return class_attrs
def _reset_lazy_submodule_refs(module):
module_class = type(module)
for entry in _DICT_DELETION:
names = getattr(module_class, entry)
except AttributeError:
for name, submodule in names.items():
super(LazyModule, module).__setattr__(name, submodule)
def _reset_lazymodule(module, class_attrs):
module_class = type(module)
del module_class.__getattribute__
del module_class.__setattr__
del module_class._LOADING
except AttributeError:
for attr in _CLS_ATTRS:
setattr(module_class, attr, class_attrs[attr])
except KeyError:
def _load_module(module):
module_class = type(module)
if not issubclass(module_class, LazyModule):
raise TypeError('Not an instance of LazyModule')
with _ImportLockContext(): with _ImportLockContext():
parent, _, module_name = module.__name__.rpartition('.') parts = module_name.split('.')
if not hasattr(module_class, '_lazy_import_error_msgs'): base_name = ''
return for part in parts:
module_class._LOADING = True if base_name:
try: base_name += '.'
if parent: base_name += part
setattr(sys.modules[parent], module_name, module) if base_name in _not_yet_imported_lazy_module_names:
if not hasattr(module_class, '_LOADING'): # This level is a lazy module, and it has not yet been loaded. Load it!
return _not_yet_imported_lazy_module_names.remove(base_name)
cached_data = _clean_lazymodule(module) module = sys.modules[base_name]
try: # Restore __getattribute__ and __setattr__ to original functionality
reload_module(module) module_class = type(module)
except Exception: module_class.__getattribute__ = ModuleType.__getattribute__
_reset_lazymodule(module, cached_data) module_class.__setattr__ = ModuleType.__setattr__
raise # Remove the fake __file__ attribute we set initially
else: ModuleType.__delattr__(module, '__file__')
delattr(module_class, '_LOADING') # Actually import the module
_reset_lazy_submodule_refs(module) try:
except (AttributeError, ImportError): importlib.reload(module)
msg = module_class._lazy_import_error_msgs['msg'] except Exception as e:
raise_from(ImportError(msg.format(**module_class._lazy_import_error_strings)), None) # Wrap the exception, to avoid potential exception misclassification.
err_msg = f'Error occured during loading of lazy module {base_name}: {e}'
raise LazyImportError(err_msg)
def _lazy_module(module_name, error_strings):
with _ImportLockContext():
full_module_name = module_name def _create_lazy_module(module_name):
full_submodule_name = None '''
submodule_name = '' Create a dedicated class and instantiate it, and adds it to sys.modules
while module_name: '''
try: class _LazyModuleSubclass(_LazyModule):
module = sys.modules[module_name] def __repr__(self):
module_name = '' return 'Lazily-loaded module %s' % self.__name__
except KeyError: module = _LazyModuleSubclass(module_name)
err_strs = error_strings.copy() ModuleType.__setattr__(module, '__file__', None)
err_strs.setdefault('module', module_name) sys.modules[module_name] = module
class _LazyModule(LazyModule): return module
_lazy_import_error_msgs = {'msg': err_strs.pop('msg')}
msg_callable = err_strs.pop('msg_callable', None)
if msg_callable: def lazy_module(module_name):
_lazy_import_error_msgs['msg_callable'] = msg_callable '''
_lazy_import_error_strings = err_strs Call this to "lazily" import a module. Subsequent calls to import will succeed
_lazy_import_callables = {} immediately, without the module actually being imported. The module is imported
_lazy_import_submodules = {} first when it is actually used (by accessing an attribute which requires the
module to really be imported).
def __repr__(self):
return 'Lazily-loaded module %s' % self.__name__ The lazy import functionality can also be disabled using the setting
nomad.config.enable_lazy_import = False
_LazyModule.__name__ = 'module' When disabled, this method does nothing, and modules are imported "as usual".
module = sys.modules[module_name] = _LazyModule(module_name) '''
if not config.enable_lazy_import:
if full_submodule_name: return
submodule = sys.modules[full_submodule_name]
ModuleType.__setattr__(module, submodule_name, submodule) if module_name not in sys.modules:
_LazyModule._lazy_import_submodules[submodule_name] = submodule # Create a lazy module object and add it, without really loading it.
# Also add a lazy module object for all parent modules, if needed.
full_submodule_name = module_name with _ImportLockContext():
module_name, _, submodule_name = module_name.rpartition('.') module = _create_lazy_module(module_name)
while True:
return sys.modules[full_module_name] parent_module_name, _, submodule_name = module_name.rpartition('.')
if not parent_module_name:
def lazy_module(module_name, level='leaf'): # Fetch or create parent_module
module_base_name = module_name.partition('.')[0] if parent_module_name in sys.modules:
error_strings = {} parent_module = sys.modules[parent_module_name]
try: parent_was_already_created = True
caller = sys._getframe(3).f_globals['__name__'] else:
except AttributeError: parent_module = _create_lazy_module(parent_module_name)
caller = 'Python' parent_was_already_created = False
error_strings.setdefault('caller', caller) # Set module as an attribute on the parent_module
error_strings.setdefault('install_name', module_base_name) ModuleType.__setattr__(parent_module, submodule_name, module)
error_strings.setdefault('msg', _MSG) if parent_was_already_created:
module = _lazy_module(module_name, error_strings) # Parent had to be lazy loaded too -> we need to also check parent's parent.
module = parent_module
if level == 'base': module_name = parent_module_name
return sys.modules[module_base_name] return
elif level == 'leaf':
return module
raise ValueError('Must be base or leaf')
...@@ -32,7 +32,6 @@ and .yaml files. This is done automatically on import. The precedence is env ove ...@@ -32,7 +32,6 @@ and .yaml files. This is done automatically on import. The precedence is env ove
over defaults. over defaults.
.. autoclass:: nomad.config.NomadConfig .. autoclass:: nomad.config.NomadConfig
.. autofunction:: nomad.config.apply
.. autofunction:: nomad.config.load_config .. autofunction:: nomad.config.load_config
''' '''
...@@ -315,6 +314,7 @@ reprocess_rematch = True ...@@ -315,6 +314,7 @@ reprocess_rematch = True
process_reuse_parser = True process_reuse_parser = True
metadata_file_name = 'nomad' metadata_file_name = 'nomad'
metadata_file_extensions = ('json', 'yaml', 'yml') metadata_file_extensions = ('json', 'yaml', 'yml')
enable_lazy_import = True
def normalize_loglevel(value, default_level=logging.INFO): def normalize_loglevel(value, default_level=logging.INFO):
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment