diff --git a/nomad/app/api/auth.py b/nomad/app/api/auth.py
index 61b1fbc2399ccca2b5390426bb6308c730b88cd9..4ad06ff438699d8350702c0802b8f0cc919ec91e 100644
--- a/nomad/app/api/auth.py
+++ b/nomad/app/api/auth.py
@@ -40,7 +40,7 @@ import hashlib
import uuid
from nomad import config, processing, utils, infrastructure, datamodel
-from nomad.metainfo.flask_restplus import generate_flask_restplus_model
+from nomad.metainfo.flask_extension import generate_flask_restplus_model
from .api import api
diff --git a/nomad/app/api/dataset.py b/nomad/app/api/dataset.py
index 5adb190e15f8c0e15f32c724eed877000cbf1c1b..4b96094acab5ffc691b66696729c1c3ddaeca1cd 100644
--- a/nomad/app/api/dataset.py
+++ b/nomad/app/api/dataset.py
@@ -18,7 +18,7 @@ import re
from nomad import utils, processing as proc
from nomad.datamodel import Dataset
-from nomad.metainfo.flask_restplus import generate_flask_restplus_model
+from nomad.metainfo.flask_extension import generate_flask_restplus_model
from nomad.doi import DOI
from nomad.app import common
diff --git a/nomad/app/api/repo.py b/nomad/app/api/repo.py
index be7de1e14b9ed60bd95900a8d02c02f48219a80d..b83f9322bc4ceefe4140bdfa5b95418773ee5bee 100644
--- a/nomad/app/api/repo.py
+++ b/nomad/app/api/repo.py
@@ -26,6 +26,7 @@ import elasticsearch.helpers
from datetime import datetime
from nomad import search, utils, datamodel, processing as proc, infrastructure
+from nomad.metainfo import search_extension
from nomad.datamodel import Dataset, User, EditableUserMetadata
from nomad.app import common
from nomad.app.common import RFC3339DateTime, DotKeyNested
@@ -54,7 +55,7 @@ class RepoCalcResource(Resource):
Calcs are references via *upload_id*, *calc_id* pairs.
'''
try:
- calc = search.Entry.get(calc_id)
+ calc = search.entry_document.get(calc_id)
except NotFoundError:
abort(404, message='There is no calculation %s/%s' % (upload_id, calc_id))
@@ -81,12 +82,12 @@ _search_request_parser.add_argument(
_search_request_parser.add_argument(
'metrics', type=str, action='append', help=(
'Metrics to aggregate over all quantities and their values as comma separated list. '
- 'Possible values are %s.' % ', '.join(search.metrics_names)))
+ 'Possible values are %s.' % ', '.join(search_extension.metrics.keys())))
_search_request_parser.add_argument(
'statistics', type=bool, help=('Return statistics.'))
_search_request_parser.add_argument(
'exclude', type=str, action='split', help='Excludes the given keys in the returned data.')
-for group_name in search.groups:
+for group_name in search_extension.groups:
_search_request_parser.add_argument(
group_name, type=bool, help=('Return %s group data.' % group_name))
_search_request_parser.add_argument(
@@ -98,15 +99,15 @@ _repo_calcs_model_fields = {
'A dict with all statistics. Each statistic is dictionary with a metrics dict as '
'value and quantity value as key. The possible metrics are code runs(calcs), %s. '
'There is a pseudo quantity "total" with a single value "all" that contains the '
- ' metrics over all results. ' % ', '.join(search.metrics_names)))}
+ ' metrics over all results. ' % ', '.join(search_extension.metrics.keys())))}
-for group_name in search.groups:
+for group_name in search_extension.groups:
_repo_calcs_model_fields[group_name] = (DotKeyNested if '.' in group_name else fields.Nested)(api.model('RepoGroup', {
'after': fields.String(description='The after value that can be used to retrieve the next %s.' % group_name),
'values': fields.Raw(description='A dict with %s as key. The values are dicts with "total" and "examples" keys.' % group_name)
}), skip_none=True)
-for qualified_name, quantity in search.search_quantities.items():
+for qualified_name, quantity in search_extension.search_quantities.items():
_repo_calcs_model_fields[qualified_name] = fields.Raw(
description=quantity.description, allow_null=True, skip_none=True)
@@ -170,7 +171,7 @@ class RepoCalcsResource(Resource):
metrics: List[str] = request.args.getlist('metrics')
with_statistics = args.get('statistics', False) or \
- any(args.get(group_name, False) for group_name in search.groups)
+ any(args.get(group_name, False) for group_name in search_extension.groups)
except Exception as e:
abort(400, message='bad parameters: %s' % str(e))
@@ -189,7 +190,7 @@ class RepoCalcsResource(Resource):
abort(400, message='invalid pagination')
for metric in metrics:
- if metric not in search.metrics_names:
+ if metric not in search_extension.metrics:
abort(400, message='there is no metric %s' % metric)
if with_statistics:
@@ -197,7 +198,7 @@ class RepoCalcsResource(Resource):
additional_metrics = [
group_quantity.metric_name
- for group_name, group_quantity in search.groups.items()
+ for group_name, group_quantity in search_extension.groups.items()
if args.get(group_name, False)]
total_metrics = metrics + additional_metrics
@@ -217,7 +218,7 @@ class RepoCalcsResource(Resource):
results = search_request.execute_scrolled(scroll_id=scroll_id, size=per_page)
else:
- for group_name, group_quantity in search.groups.items():
+ for group_name, group_quantity in search_extension.groups.items():
if args.get(group_name, False):
kwargs: Dict[str, Any] = {}
if group_name == 'group_uploads':
@@ -239,7 +240,7 @@ class RepoCalcsResource(Resource):
if 'quantities' in results:
quantities = results.pop('quantities')
- for group_name, group_quantity in search.groups.items():
+ for group_name, group_quantity in search_extension.groups.items():
if args.get(group_name, False):
results[group_name] = quantities[group_quantity.qualified_name]
@@ -330,7 +331,7 @@ def edit(parsed_query: Dict[str, Any], mongo_update: Dict[str, Any] = None, re_i
if re_index:
def elastic_updates():
for calc in proc.Calc.objects(calc_id__in=calc_ids):
- entry = search.create_entry(
+ entry = datamodel.EntryMetadata.m_def.m_x('elastic').create_index_entry(
datamodel.EntryMetadata.m_from_dict(calc['metadata']))
entry = entry.to_dict(include_meta=True)
entry['_op_type'] = 'index'
diff --git a/nomad/app/api/upload.py b/nomad/app/api/upload.py
index cb04f51589a96fbf16b75489d3fbef593a642a3f..0475fd8a3c3e6ba2a6d6f2a2ff2260046562342a 100644
--- a/nomad/app/api/upload.py
+++ b/nomad/app/api/upload.py
@@ -27,7 +27,7 @@ import os
import io
from functools import wraps
-from nomad import config, utils, files, search, datamodel
+from nomad import config, utils, files, datamodel
from nomad.processing import Upload, FAILURE
from nomad.processing import ProcessAlreadyRunning
from nomad.app import common
@@ -46,7 +46,7 @@ ns = api.namespace(
class CalcMetadata(fields.Raw):
def format(self, value):
entry_metadata = datamodel.EntryMetadata.m_from_dict(value)
- return search.create_entry(entry_metadata).to_dict()
+ return datamodel.EntryMetadata.m_def.m_x('elastic').create_index_entry(entry_metadata).to_dict()
proc_model = api.model('Processing', {
diff --git a/nomad/app/optimade/endpoints.py b/nomad/app/optimade/endpoints.py
index 5f41a8f292bf282c189112b95e1fd8a0beef36e4..f492be1cb6475783f654c1c79129e4d3b01c5aed 100644
--- a/nomad/app/optimade/endpoints.py
+++ b/nomad/app/optimade/endpoints.py
@@ -17,7 +17,7 @@ from flask import request
from elasticsearch_dsl import Q
from nomad import search
-from nomad.metainfo.optimade import OptimadeEntry
+from nomad.datamodel import OptimadeEntry
from .api import api, url
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
diff --git a/nomad/app/optimade/filterparser.py b/nomad/app/optimade/filterparser.py
index a95e78100af84b5bd8a457ed596ea40919f5b95a..837e2b87532d2d9478ec98042460d27637500636 100644
--- a/nomad/app/optimade/filterparser.py
+++ b/nomad/app/optimade/filterparser.py
@@ -13,10 +13,12 @@
# limitations under the License.
from typing import Dict
+from elasticsearch_dsl import Q
+
from optimade.filterparser import LarkParser
from optimade.filtertransformers.elasticsearch import Transformer, Quantity
-from elasticsearch_dsl import Q
-from nomad.metainfo.optimade import OptimadeEntry
+
+from nomad.datamodel import OptimadeEntry
class FilterException(Exception):
@@ -27,7 +29,7 @@ class FilterException(Exception):
quantities: Dict[str, Quantity] = {
q.name: Quantity(
q.name, es_field='dft.optimade.%s' % q.name,
- elastic_mapping_type=q.m_x('search').es_mapping.__class__)
+ elastic_mapping_type=q.m_x('search').mapping.__class__)
for q in OptimadeEntry.m_def.all_quantities.values()
if 'search' in q.m_annotations}
diff --git a/nomad/cli/admin/admin.py b/nomad/cli/admin/admin.py
index 8ee72493e58939b1df9a6f5c0a2f1b8d112b694a..d72dddfb4f3fb612b5ecaaeb339b520e95e909d0 100644
--- a/nomad/cli/admin/admin.py
+++ b/nomad/cli/admin/admin.py
@@ -180,7 +180,7 @@ def index(threads, dry):
for calc in proc.Calc.objects():
eta.add()
entry = None
- entry = search.create_entry(
+ entry = datamodel.EntryMetadata.m_def.m_x('elastic').create_index_entry(
datamodel.EntryMetadata.m_from_dict(calc.metadata))
entry = entry.to_dict(include_meta=True)
entry['_op_type'] = 'index'
diff --git a/nomad/datamodel/__init__.py b/nomad/datamodel/__init__.py
index c4f152be38c377a9defbf687b879f9b393e96b16..cd336e3c6af7033d939302e665f2bd03d0426b79 100644
--- a/nomad/datamodel/__init__.py
+++ b/nomad/datamodel/__init__.py
@@ -41,11 +41,23 @@ The class :class:`EntryMetadata` is used to represent all metadata about an entr
.. autoclass:: nomad.datamodel.EntryMetadata
:members:
+
+In addition there are domain specific metadata classes:
+
+.. autoclass:: nomad.datamodel.dft.DFTMetadata
+ :members:
+
+.. autoclass:: nomad.datamodel.dft.EMSMetadata
+ :members:
+
+.. autoclass:: nomad.datamodel.OptimadeEntry
+ :members:
'''
from .dft import DFTMetadata
from .ems import EMSMetadata
-from .metainfo import Dataset, User, EditableUserMetadata, UserMetadata, EntryMetadata
+from .datamodel import Dataset, User, EditableUserMetadata, UserMetadata, EntryMetadata
+from .optimade import OptimadeEntry, Species
domains = {
'dft': {
diff --git a/nomad/datamodel/base.py b/nomad/datamodel/base.py
deleted file mode 100644
index 99705430c081d9468eec637d3538da74a3e1a34e..0000000000000000000000000000000000000000
--- a/nomad/datamodel/base.py
+++ /dev/null
@@ -1,380 +0,0 @@
-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import numpy as np
-
-from nomad import config
-
-# from .metainfo import Dataset, User, EntryMetadata
-
-
-# class DomainQuantity:
-# '''
-# This class can be used to define further details about a domain specific metadata
-# quantity.
-
-# Attributes:
-# name: The name of the quantity, also the key used to store values in
-# :class:`EntryMetadata`
-# description: A human friendly description. The description is used to define
-# the swagger documentation on the relevant API endpoints.
-# multi: Indicates a list of values. This is important for the elastic mapping.
-# order_default: Indicates that this metric should be used for the default order of
-# search results.
-# aggregations: Indicates that search aggregations (and how many) should be provided.
-# 0 (the default) means no aggregations.
-# metric: Indicates that this quantity should be used as search metric. Values need
-# to be tuples with metric name and elastic aggregation (e.g. sum, cardinality)
-# elastic_mapping: An optional elasticsearch_dsl mapping. Default is ``Keyword``.
-# elastic_search_type: An optional elasticsearch search type. Default is ``term``.
-# elastic_field: An optional elasticsearch key. Default is the name of the quantity.
-# elastic_value: A collable that takes a :class:`EntryMetadata` as input and produces the
-# value for the elastic search index.
-# argparse_action: Action to use on argparse, either append or split for multi values. Append is default.
-# '''
-
-# def __init__(
-# self, description: str = None, multi: bool = False, aggregations: int = 0,
-# order_default: bool = False, metric: Tuple[str, str] = None,
-# metadata_field: str = None, elastic_mapping: type = None,
-# elastic_search_type: str = 'term', elastic_field: str = None,
-# elastic_value: Callable[[Any], Any] = None,
-# argparse_action: str = 'append'):
-
-# self.domain: str = None
-# self._name: str = None
-# self.description = description
-# self.multi = multi
-# self.order_default = order_default
-# self.aggregations = aggregations
-# self.metric = metric
-# self.elastic_mapping = elastic_mapping
-# self.elastic_search_type = elastic_search_type
-# self.metadata_field = metadata_field
-# self.elastic_field = elastic_field
-# self.argparse_action = argparse_action
-
-# self.elastic_value = elastic_value
-# if self.elastic_value is None:
-# self.elastic_value = lambda o: o
-
-# if self.elastic_mapping is None:
-# self.elastic_mapping = Keyword(multi=self.multi)
-
-# @property
-# def name(self) -> str:
-# return self._name
-
-# @name.setter
-# def name(self, name: str) -> None:
-# self._name = name
-# if self.metadata_field is None:
-# self.metadata_field = name
-# if self.elastic_field is None:
-# self.elastic_field = self.name
-
-# @property
-# def qualified_elastic_field(self) -> str:
-# if self.domain is None:
-# return self.elastic_field
-# else:
-# return '%s.%s' % (self.domain, self.elastic_field)
-
-# @property
-# def qualified_name(self) -> str:
-# if self.domain is None:
-# return self.name
-# else:
-# return '%s.%s' % (self.domain, self.name)
-
-
-# def only_atoms(atoms):
-# numbers = [ase.data.atomic_numbers[atom] for atom in atoms]
-# only_atoms = [ase.data.chemical_symbols[number] for number in sorted(numbers)]
-# return ''.join(only_atoms)
-
-
-# class Domain:
-# '''
-# A domain defines all metadata quantities that are specific to a certain scientific
-# domain, e.g. DFT calculations, or experimental material science.
-
-# Each domain needs to define a subclass of :class:`EntryMetadata`. This
-# class has to define the necessary domain specific metadata quantities and how these
-# are filled from parser results (usually an instance of :class:LocalBackend).
-
-# Furthermore, the class method :func:`register_domain` of this ``Domain`` class has
-# to be used to register a domain with ``domain_nam``. This also allows to provide
-# further descriptions on each domain specific quantity via instance of :class:`DomainQuantity`.
-
-# While there can be multiple domains registered. Currently, only one domain can be
-# active. This active domain is define in the configuration using the ``domain_name``.
-
-# Arguments:
-# name: A name for the domain. This is used as key in the configuration ``config.domain``.
-# domain_entry_class: A subclass of :class:`EntryMetadata` that adds the
-# domain specific quantities.
-# quantities: Additional specifications for the quantities in ``domain_entry_class`` as
-# instances of :class:`DomainQuantity`.
-# metrics: Tuples of elastic field name and elastic aggregation operation that
-# can be used to create statistic values.
-# groups: Tuple of quantity name and metric that describes quantities that
-# can be used to group entries by quantity values.
-# root_sections: The name of the possible root sections for this domain.
-# metainfo_all_package: The name of the full metainfo package for this domain.
-# '''
-# instances: Dict[str, 'Domain'] = {}
-
-# base_quantities = dict(
-# authors=DomainQuantity(
-# elastic_field='authors.name.keyword', multi=True, aggregations=1000,
-# description=(
-# 'Search for the given author. Exact keyword matches in the form "Lastname, '
-# 'Firstname".')),
-# uploader_id=DomainQuantity(
-# elastic_field='uploader.user_id', multi=False, aggregations=5,
-# description=('Search for the given uploader id.')),
-# uploader_name=DomainQuantity(
-# elastic_field='uploader.name.keyword', multi=False,
-# description=('Search for the exact uploader\'s full name')),
-# comment=DomainQuantity(
-# elastic_search_type='match', multi=True,
-# description='Search within the comments. This is a text search ala google.'),
-# paths=DomainQuantity(
-# elastic_search_type='match', elastic_field='files', multi=True,
-# description='Search for elements in one of the file paths. The paths are split at all "/".'),
-# files=DomainQuantity(
-# elastic_field='files.keyword', multi=True,
-# description='Search for exact file name with full path.'),
-# quantities=DomainQuantity(
-# multi=True,
-# description='Search for the existence of a certain meta-info quantity'),
-# upload_id=DomainQuantity(
-# description='Search for the upload_id.',
-# multi=True, argparse_action='split', elastic_search_type='terms'),
-# upload_time=DomainQuantity(
-# description='Search for the exact upload time.', elastic_search_type='terms'),
-# upload_name=DomainQuantity(
-# description='Search for the upload_name.',
-# multi=True, argparse_action='split', elastic_search_type='terms'),
-# calc_id=DomainQuantity(
-# description='Search for the calc_id.',
-# multi=True, argparse_action='split', elastic_search_type='terms'),
-# pid=DomainQuantity(
-# description='Search for the pid.',
-# multi=True, argparse_action='split', elastic_search_type='terms'),
-# raw_id=DomainQuantity(
-# description='Search for the raw_id.',
-# multi=True, argparse_action='split', elastic_search_type='terms'),
-# mainfile=DomainQuantity(
-# description='Search for the mainfile.',
-# multi=True, argparse_action='append', elastic_search_type='terms'),
-# external_id=DomainQuantity(
-# description='External user provided id. Does not have to be unique necessarily.',
-# multi=True, argparse_action='split', elastic_search_type='terms'),
-# calc_hash=DomainQuantity(
-# description='Search for the entries hash.',
-# multi=True, argparse_action='split', elastic_search_type='terms'),
-# dataset=DomainQuantity(
-# elastic_field='datasets.name', multi=True, elastic_search_type='match',
-# description='Search for a particular dataset by name.'),
-# dataset_id=DomainQuantity(
-# elastic_field='datasets.id', multi=True,
-# description='Search for a particular dataset by its id.'),
-# doi=DomainQuantity(
-# elastic_field='datasets.doi', multi=True,
-# description='Search for a particular dataset by doi (incl. http://dx.doi.org).'),
-# formula=DomainQuantity(
-# 'The chemical (hill) formula of the simulated system.',
-# order_default=True),
-# atoms=DomainQuantity(
-# 'The atom labels of all atoms in the simulated system.',
-# aggregations=len(ase.data.chemical_symbols), multi=True),
-# only_atoms=DomainQuantity(
-# 'The atom labels concatenated in species-number order. Used with keyword search '
-# 'to facilitate exclusive searches.',
-# elastic_value=only_atoms, metadata_field='atoms', multi=True),
-# n_atoms=DomainQuantity(
-# 'Number of atoms in the simulated system',
-# elastic_mapping=Integer()))
-
-# base_metrics = dict(
-# datasets=('dataset_id', 'cardinality'),
-# uploads=('upload_id', 'cardinality'),
-# uploaders=('uploader_name', 'cardinality'),
-# authors=('authors', 'cardinality'),
-# unique_entries=('calc_hash', 'cardinality'))
-
-# base_groups = dict(
-# datasets=('dataset_id', 'datasets'),
-# uploads=('upload_id', 'uploads'))
-
-# @classmethod
-# def get_quantity(cls, name_spec) -> DomainQuantity:
-# '''
-# Returns the quantity definition for the given quantity name. The name can be the
-# qualified name (``domain.quantity``) or in Django-style (``domain__quantity``).
-# '''
-# qualified_name = name_spec.replace('__', '.')
-# split_name = qualified_name.split('.')
-# if len(split_name) == 1:
-# return cls.base_quantities[split_name[0]]
-# elif len(split_name) == 2:
-# return cls.instances[split_name[0]].quantities[split_name[1]]
-# else:
-# assert False, 'qualified quantity name depth must be 2 max'
-
-# @classmethod
-# def all_quantities(cls) -> Iterable[DomainQuantity]:
-# return set([quantity for domain in cls.instances.values() for quantity in domain.quantities.values()])
-
-# def __init__(
-# self, name: str, domain_entry_class: Type[EntryMetadata],
-# quantities: Dict[str, DomainQuantity],
-# metrics: Dict[str, Tuple[str, str]],
-# groups: Dict[str, Tuple[str, str]],
-# default_statistics: List[str],
-# root_sections=['section_run', 'section_entry_info'],
-# metainfo_all_package='all.nomadmetainfo.json') -> None:
-
-# domain_quantities = quantities
-
-# Domain.instances[name] = self
-
-# self.name = name
-# self.domain_entry_class = domain_entry_class
-# self.domain_quantities: Dict[str, DomainQuantity] = {}
-# self.root_sections = root_sections
-# self.metainfo_all_package = metainfo_all_package
-# self.default_statistics = default_statistics
-
-# # TODO
-# return
-
-# reference_domain_calc = EntryMetadata(domain=name)
-# reference_general_calc = EntryMetadata(domain=None)
-
-# # add non specified quantities from additional metadata class fields
-# for quantity_name in reference_domain_calc.__dict__.keys():
-# if not hasattr(reference_general_calc, quantity_name):
-# quantity = domain_quantities.get(quantity_name, None)
-# if quantity is None:
-# domain_quantities[quantity_name] = DomainQuantity()
-
-# # ensure domain quantity names and domains
-# for quantity_name, quantity in domain_quantities.items():
-# quantity.domain = name
-# quantity.name = quantity_name
-
-# # add domain prefix to domain metrics and groups
-# domain_metrics = {
-# '%s.%s' % (name, key): (quantities[quantity].qualified_elastic_field, es_op)
-# for key, (quantity, es_op) in metrics.items()}
-# domain_groups = {
-# '%s.%s' % (name, key): (quantities[quantity].qualified_name, '%s.%s' % (name, metric))
-# for key, (quantity, metric) in groups.items()}
-
-# # add all domain quantities
-# for quantity_name, quantity in domain_quantities.items():
-# self.domain_quantities[quantity.name] = quantity
-
-# # update the multi status from an example value
-# if quantity.metadata_field in reference_domain_calc.__dict__:
-# quantity.multi = isinstance(
-# reference_domain_calc.__dict__[quantity.metadata_field], list)
-
-# assert not hasattr(reference_general_calc, quantity_name), \
-# 'quantity overrides general non domain quantity: %s' % quantity_name
-
-# # construct search quantities from base and domain quantities
-# self.quantities = dict(**Domain.base_quantities)
-# for quantity_name, quantity in self.quantities.items():
-# quantity.name = quantity_name
-# self.quantities.update(self.domain_quantities)
-
-# assert any(quantity.order_default for quantity in Domain.instances[name].quantities.values()), \
-# 'you need to define a order default quantity'
-
-# # construct metrics from base and domain metrics
-# self.metrics = dict(**Domain.base_metrics)
-# self.metrics.update(**domain_metrics)
-# self.groups = dict(**Domain.base_groups)
-# self.groups.update(**domain_groups)
-
-# @property
-# def metrics_names(self) -> Iterable[str]:
-# ''' Just the names of all metrics. '''
-# return list(self.metrics.keys())
-
-# @property
-# def aggregations(self) -> Dict[str, int]:
-# '''
-# The search aggregations and the default maximum number of calculated buckets. See also
-# :func:`nomad.search.aggregations`.
-# '''
-# return {
-# quantity.name: quantity.aggregations
-# for quantity in self.quantities.values()
-# if quantity.aggregations > 0
-# }
-
-# @property
-# def aggregations_names(self) -> Iterable[str]:
-# ''' Just the names of all metrics. '''
-# return list(self.aggregations.keys())
-
-# @property
-# def order_default_quantity(self) -> str:
-# for quantity in self.quantities.values():
-# if quantity.order_default:
-# return quantity.qualified_name
-
-# assert False, 'each domain must defina an order_default quantity'
-
-
-def get_optional_backend_value(backend, key, section, unavailable_value=None, logger=None):
- # Section is section_system, section_symmetry, etc...
- val = None # Initialize to None, so we can compare section values.
- # Loop over the sections with the name section in the backend.
- for section_index in backend.get_sections(section):
- if section == 'section_system':
- try:
- if not backend.get_value('is_representative', section_index):
- continue
- except (KeyError, IndexError):
- continue
-
- try:
- new_val = backend.get_value(key, section_index)
- except (KeyError, IndexError):
- new_val = None
-
- # Compare values from iterations.
- if val is not None and new_val is not None:
- if val.__repr__() != new_val.__repr__() and logger:
- logger.warning(
- 'The values for %s differ between different %s: %s vs %s' %
- (key, section, str(val), str(new_val)))
-
- val = new_val if new_val is not None else val
-
- if val is None and logger:
- logger.warning(
- 'The values for %s where not available in any %s' % (key, section))
- return unavailable_value if unavailable_value is not None else config.services.unavailable_value
- else:
- if isinstance(val, np.generic):
- return val.item()
-
- return val
diff --git a/nomad/datamodel/common.py b/nomad/datamodel/common.py
new file mode 100644
index 0000000000000000000000000000000000000000..706b59f23d8523e769cc4a06052e05327c0df056
--- /dev/null
+++ b/nomad/datamodel/common.py
@@ -0,0 +1,54 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy as np
+
+from nomad import config
+
+
+def get_optional_backend_value(backend, key, section, unavailable_value=None, logger=None):
+ # Section is section_system, section_symmetry, etc...
+ val = None # Initialize to None, so we can compare section values.
+ # Loop over the sections with the name section in the backend.
+ for section_index in backend.get_sections(section):
+ if section == 'section_system':
+ try:
+ if not backend.get_value('is_representative', section_index):
+ continue
+ except (KeyError, IndexError):
+ continue
+
+ try:
+ new_val = backend.get_value(key, section_index)
+ except (KeyError, IndexError):
+ new_val = None
+
+ # Compare values from iterations.
+ if val is not None and new_val is not None:
+ if val.__repr__() != new_val.__repr__() and logger:
+ logger.warning(
+ 'The values for %s differ between different %s: %s vs %s' %
+ (key, section, str(val), str(new_val)))
+
+ val = new_val if new_val is not None else val
+
+ if val is None and logger:
+ logger.warning(
+ 'The values for %s where not available in any %s' % (key, section))
+ return unavailable_value if unavailable_value is not None else config.services.unavailable_value
+ else:
+ if isinstance(val, np.generic):
+ return val.item()
+
+ return val
diff --git a/nomad/datamodel/metainfo.py b/nomad/datamodel/datamodel.py
similarity index 86%
rename from nomad/datamodel/metainfo.py
rename to nomad/datamodel/datamodel.py
index 24f064b544f53e541964e669da57c8fa14782d3c..48020c8b07cc47b27dcbea66bd1eb298eb763ce9 100644
--- a/nomad/datamodel/metainfo.py
+++ b/nomad/datamodel/datamodel.py
@@ -12,18 +12,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-'''
-This duplicates functionality for .base.py. It represents first pieces of a transition
-towards using the new metainfo system for all repository metadata.
-'''
+''' All generic entry metadata and related classes. '''
+
from typing import Dict, Any
from cachetools import cached, TTLCache
from elasticsearch_dsl import Keyword, Text, analyzer, tokenizer
import ase.data
from nomad import metainfo, config
-from nomad.metainfo.search import SearchQuantity
-import nomad.metainfo.mongoengine
+from nomad.metainfo.search_extension import Search
+from nomad.metainfo.elastic_extension import ElasticDocument
+import nomad.metainfo.mongoengine_extension
from .dft import DFTMetadata
from .ems import EMSMetadata
@@ -62,12 +61,12 @@ class User(metainfo.MSection):
user_id = metainfo.Quantity(
type=str,
a_me=dict(primary_key=True),
- a_search=SearchQuantity())
+ a_search=Search())
name = metainfo.Quantity(
type=str,
derived=lambda user: ('%s %s' % (user.first_name, user.last_name)).strip(),
- a_search=SearchQuantity(es_mapping=Text(fields={'keyword': Keyword()})))
+ a_search=Search(mapping=Text(fields={'keyword': Keyword()})))
first_name = metainfo.Quantity(type=str)
last_name = metainfo.Quantity(type=str)
@@ -75,7 +74,7 @@ class User(metainfo.MSection):
type=str,
a_me=dict(index=True),
a_elastic=dict(mapping=Keyword), # TODO remove?
- a_search=SearchQuantity())
+ a_search=Search())
username = metainfo.Quantity(type=str)
affiliation = metainfo.Quantity(type=str)
@@ -149,25 +148,25 @@ class Dataset(metainfo.MSection):
dataset_id = metainfo.Quantity(
type=str,
a_me=dict(primary_key=True),
- a_search=SearchQuantity())
+ a_search=Search())
name = metainfo.Quantity(
type=str,
a_me=dict(index=True),
- a_search=SearchQuantity())
+ a_search=Search())
user_id = metainfo.Quantity(
type=str,
a_me=dict(index=True))
doi = metainfo.Quantity(
type=str,
a_me=dict(index=True),
- a_search=SearchQuantity())
+ a_search=Search())
pid = metainfo.Quantity(
type=str,
a_me=dict(index=True))
created = metainfo.Quantity(
type=metainfo.Datetime,
a_me=dict(index=True),
- a_search=SearchQuantity())
+ a_search=Search())
class DatasetReference(metainfo.Reference):
@@ -248,72 +247,75 @@ class EntryMetadata(metainfo.MSection):
upload_time: The time that this entry was uploaded
datasets: Ids of all datasets that this entry appears in
'''
+ m_def = metainfo.Section(a_elastic=ElasticDocument(
+ index_name=config.elastic.index_name, id=lambda x: x.calc_id))
+
upload_id = metainfo.Quantity(
type=str,
description='A random UUID that uniquely identifies the upload of the entry.',
- a_search=SearchQuantity(
+ a_search=Search(
many_or='append', group='uploads_grouped', metric_name='uploads', metric='cardinality'))
calc_id = metainfo.Quantity(
type=str,
description='A unique ID based on the upload id and entry\'s mainfile.',
- a_search=SearchQuantity(many_or='append'))
+ a_search=Search(many_or='append'))
calc_hash = metainfo.Quantity(
type=str,
description='A raw file content based checksum/hash.',
- a_search=SearchQuantity(
+ a_search=Search(
many_or='append', metric_name='unique_entries', metric='cardinality'))
mainfile = metainfo.Quantity(
type=str,
description='The upload relative mainfile path.',
a_search=[
- SearchQuantity(
+ Search(
description='Search within the mainfile path.',
- es_mapping=Text(multi=True, analyzer=path_analyzer, fields={'keyword': Keyword()}),
- many_or='append', es_quantity='mainfile.keyword'),
- SearchQuantity(
+ mapping=Text(multi=True, analyzer=path_analyzer, fields={'keyword': Keyword()}),
+ many_or='append', search_field='mainfile.keyword'),
+ Search(
description='Search for the exact mainfile.',
- many_and='append', name='mainfile_path', es_quantity='mainfile.keyword')])
+ many_and='append', name='mainfile_path', search_field='mainfile.keyword')])
files = metainfo.Quantity(
type=str, shape=['0..*'],
description='The entries raw file paths relative to its upload.',
a_search=[
- SearchQuantity(
+ Search(
description='Search within the paths.', name='path',
- es_mapping=Text(
+ mapping=Text(
multi=True, analyzer=path_analyzer, fields={'keyword': Keyword()})
),
- SearchQuantity(
+ Search(
description='Search for exact paths.',
- many_or='append', name='files', es_quantity='files.keyword')])
+ many_or='append', name='files', search_field='files.keyword')])
pid = metainfo.Quantity(
type=int,
description='The unique, sequentially enumerated, integer persistent identifier',
- a_search=SearchQuantity(many_or='append'))
+ a_search=Search(many_or='append'))
raw_id = metainfo.Quantity(
type=str,
description='A raw format specific id that was acquired from the files of this entry',
- a_search=SearchQuantity(many_or='append'))
+ a_search=Search(many_or='append'))
domain = metainfo.Quantity(
type=metainfo.MEnum('dft', 'ems'),
description='The material science domain',
- a_search=SearchQuantity())
+ a_search=Search())
published = metainfo.Quantity(
type=bool, default=False,
description='Indicates if the entry is published',
- a_search=SearchQuantity())
+ a_search=Search())
processed = metainfo.Quantity(
type=bool, default=False,
description='Indicates that the entry is successfully processed.',
- a_search=SearchQuantity())
+ a_search=Search())
last_processing = metainfo.Quantity(
type=metainfo.Datetime,
@@ -322,37 +324,37 @@ class EntryMetadata(metainfo.MSection):
nomad_version = metainfo.Quantity(
type=str,
description='The NOMAD version used for the last processing attempt.',
- a_search=SearchQuantity(many_or='append'))
+ a_search=Search(many_or='append'))
nomad_commit = metainfo.Quantity(
type=str,
description='The NOMAD commit used for the last processing attempt.',
- a_search=SearchQuantity(many_or='append'))
+ a_search=Search(many_or='append'))
parser_name = metainfo.Quantity(
type=str,
description='The NOMAD parser used for the last processing attempt.',
- a_search=SearchQuantity(many_or='append'))
+ a_search=Search(many_or='append'))
comment = metainfo.Quantity(
type=str, categories=[UserMetadata, EditableUserMetadata],
description='A user provided comment.',
- a_search=SearchQuantity(es_mapping=Text()))
+ a_search=Search(mapping=Text()))
references = metainfo.Quantity(
type=str, shape=['0..*'], categories=[UserMetadata, EditableUserMetadata],
description='User provided references (URLs).',
- a_search=SearchQuantity())
+ a_search=Search())
uploader = metainfo.Quantity(
type=user_reference, categories=[UserMetadata],
description='The uploader of the entry',
a_flask=dict(admin_only=True, verify=User),
a_search=[
- SearchQuantity(
+ Search(
description='Search uploader with exact names.',
metric_name='uploaders', metric='cardinality',
- many_or='append', es_quantity='uploader.name.keyword'),
- SearchQuantity(
- name='uploader_id', es_quantity='uploader.user_id')
+ many_or='append', search_field='uploader.name.keyword'),
+ Search(
+ name='uploader_id', search_field='uploader.user_id')
])
coauthors = metainfo.Quantity(
@@ -364,10 +366,10 @@ class EntryMetadata(metainfo.MSection):
type=user_reference, shape=['0..*'],
description='All authors (uploader and co-authors).',
derived=lambda entry: ([entry.uploader] if entry.uploader is not None else []) + entry.coauthors,
- a_search=SearchQuantity(
+ a_search=Search(
description='Search authors with exact names.',
metric='cardinality',
- many_or='append', es_quantity='authors.name.keyword', statistic_size=1000))
+ many_or='append', search_field='authors.name.keyword', statistic_size=1000))
shared_with = metainfo.Quantity(
type=user_reference, shape=['0..*'], default=[], categories=[UserMetadata, EditableUserMetadata],
@@ -378,25 +380,25 @@ class EntryMetadata(metainfo.MSection):
type=user_reference, shape=['0..*'],
description='All owner (uploader and shared with users).',
derived=lambda entry: ([entry.uploader] if entry.uploader is not None else []) + entry.shared_with,
- a_search=SearchQuantity(
+ a_search=Search(
description='Search owner with exact names.',
- many_or='append', es_quantity='owners.name.keyword'))
+ many_or='append', search_field='owners.name.keyword'))
with_embargo = metainfo.Quantity(
type=bool, default=False, categories=[UserMetadata, EditableUserMetadata],
description='Indicated if this entry is under an embargo',
- a_search=SearchQuantity())
+ a_search=Search())
upload_time = metainfo.Quantity(
type=metainfo.Datetime, categories=[UserMetadata],
description='The datetime this entry was uploaded to nomad',
a_flask=dict(admin_only=True),
- a_search=SearchQuantity(order_default=True))
+ a_search=Search(order_default=True))
upload_name = metainfo.Quantity(
type=str, categories=[UserMetadata],
description='The user provided upload name',
- a_search=SearchQuantity(many_or='append'))
+ a_search=Search(many_or='append'))
datasets = metainfo.Quantity(
type=dataset_reference, shape=['0..*'], default=[],
@@ -404,11 +406,11 @@ class EntryMetadata(metainfo.MSection):
description='A list of user curated datasets this entry belongs to.',
a_flask=dict(verify=Dataset),
a_search=[
- SearchQuantity(
- es_quantity='datasets.name', many_or='append',
+ Search(
+ search_field='datasets.name', many_or='append',
description='Search for a particular dataset by exact name.'),
- SearchQuantity(
- name='dataset_id', es_quantity='datasets.dataset_id', many_or='append',
+ Search(
+ name='dataset_id', search_field='datasets.dataset_id', many_or='append',
group='datasets_grouped',
metric='cardinality', metric_name='datasets',
description='Search for a particular dataset by its id.')])
@@ -416,34 +418,34 @@ class EntryMetadata(metainfo.MSection):
external_id = metainfo.Quantity(
type=str, categories=[UserMetadata],
description='A user provided external id.',
- a_search=SearchQuantity(many_or='split'))
+ a_search=Search(many_or='split'))
last_edit = metainfo.Quantity(
type=metainfo.Datetime, categories=[UserMetadata],
description='The datetime the user metadata was edited last.',
- a_search=SearchQuantity())
+ a_search=Search())
formula = metainfo.Quantity(
type=str, categories=[DomainMetadata],
description='A (reduced) chemical formula.',
- a_search=SearchQuantity())
+ a_search=Search())
atoms = metainfo.Quantity(
type=str, shape=['n_atoms'], default=[], categories=[DomainMetadata],
description='The atom labels of all atoms of the entry\'s material.',
- a_search=SearchQuantity(
+ a_search=Search(
many_and='append', default_statistic=True, statistic_size=len(ase.data.chemical_symbols)))
only_atoms = metainfo.Quantity(
type=str, categories=[DomainMetadata],
description='The atom labels concatenated in order-number order.',
derived=lambda entry: _only_atoms(entry.atoms),
- a_search=SearchQuantity(many_and='append', derived=_only_atoms))
+ a_search=Search(many_and='append', derived=_only_atoms))
n_atoms = metainfo.Quantity(
type=int, categories=[DomainMetadata],
description='The number of atoms in the entry\'s material',
- a_search=SearchQuantity())
+ a_search=Search())
ems = metainfo.SubSection(sub_section=EMSMetadata, a_search='ems')
dft = metainfo.SubSection(sub_section=DFTMetadata, a_search='dft')
@@ -460,5 +462,5 @@ class EntryMetadata(metainfo.MSection):
domain_section.apply_domain_metadata(backend)
-nomad.metainfo.mongoengine.init_section(User)
-nomad.metainfo.mongoengine.init_section(Dataset)
+nomad.metainfo.mongoengine_extension.init_section(User)
+nomad.metainfo.mongoengine_extension.init_section(Dataset)
diff --git a/nomad/datamodel/dft.py b/nomad/datamodel/dft.py
index 7eac3a6cbba47ec8508b544db8c673b8f8584b20..e4d6d892000dbab6f3839374048e4fb836d0f701 100644
--- a/nomad/datamodel/dft.py
+++ b/nomad/datamodel/dft.py
@@ -21,10 +21,11 @@ import re
from nomadcore.local_backend import ParserEvent
from nomad import utils, config
-from nomad.metainfo import optimade, MSection, Section, Quantity, MEnum, SubSection
-from nomad.metainfo.search import SearchQuantity
+from nomad.metainfo import MSection, Section, Quantity, MEnum, SubSection
+from nomad.metainfo.search_extension import Search
-from .base import get_optional_backend_value
+from .common import get_optional_backend_value
+from .optimade import OptimadeEntry
xc_treatments = {
@@ -77,15 +78,15 @@ class Label(MSection):
source: The source that this label was taken from.
'''
- label = Quantity(type=str, a_search=SearchQuantity())
+ label = Quantity(type=str, a_search=Search())
type = Quantity(type=MEnum(
'compound_class', 'classification', 'prototype', 'prototype_id'),
- a_search=SearchQuantity())
+ a_search=Search())
source = Quantity(
type=MEnum('springer', 'aflow_prototype_library'),
- a_search=SearchQuantity())
+ a_search=Search())
class DFTMetadata(MSection):
@@ -94,75 +95,75 @@ class DFTMetadata(MSection):
basis_set = Quantity(
type=str, default='not processed',
description='The used basis set functions.',
- a_search=SearchQuantity(statistic_size=20, default_statistic=True))
+ a_search=Search(statistic_size=20, default_statistic=True))
xc_functional = Quantity(
type=str, default='not processed',
description='The libXC based xc functional classification used in the simulation.',
- a_search=SearchQuantity(statistic_size=20, default_statistic=True))
+ a_search=Search(statistic_size=20, default_statistic=True))
system = Quantity(
type=str, default='not processed',
description='The system type of the simulated system.',
- a_search=SearchQuantity(default_statistic=True))
+ a_search=Search(default_statistic=True))
crystal_system = Quantity(
type=str, default='not processed',
description='The crystal system type of the simulated system.',
- a_search=SearchQuantity(default_statistic=True))
+ a_search=Search(default_statistic=True))
spacegroup = Quantity(
type=int, default='not processed',
description='The spacegroup of the simulated system as number.',
- a_search=SearchQuantity())
+ a_search=Search())
spacegroup_symbol = Quantity(
type=str, default='not processed',
description='The spacegroup as international short symbol.',
- a_search=SearchQuantity())
+ a_search=Search())
code_name = Quantity(
type=str, default='not processed',
description='The name of the used code.',
- a_search=SearchQuantity(statistic_size=40, default_statistic=True))
+ a_search=Search(statistic_size=40, default_statistic=True))
code_version = Quantity(
type=str, default='not processed',
description='The version of the used code.',
- a_search=SearchQuantity())
+ a_search=Search())
n_geometries = Quantity(
type=int, description='Number of unique geometries.',
- a_search=SearchQuantity(metric_name='geometries', metric='sum'))
+ a_search=Search(metric_name='geometries', metric='sum'))
n_calculations = Quantity(
type=int,
description='Number of single configuration calculation sections',
- a_search=SearchQuantity(metric_name='calculations', metric='sum'))
+ a_search=Search(metric_name='calculations', metric='sum'))
n_total_energies = Quantity(
type=int, description='Number of total energy calculations',
- a_search=SearchQuantity(metric_name='total_energies', metric='sum'))
+ a_search=Search(metric_name='total_energies', metric='sum'))
n_quantities = Quantity(
type=int, description='Number of metainfo quantities parsed from the entry.',
- a_search=SearchQuantity(metric='sum', metric_name='quantities'))
+ a_search=Search(metric='sum', metric_name='quantities'))
quantities = Quantity(
type=str, shape=['0..*'],
description='All quantities that are used by this entry.',
- a_search=SearchQuantity(
+ a_search=Search(
metric_name='distinct_quantities', metric='cardinality', many_and='append'))
geometries = Quantity(
type=str, shape=['0..*'],
description='Hashes for each simulated geometry',
- a_search=SearchQuantity(metric_name='unique_geometries', metric='cardinality'))
+ a_search=Search(metric_name='unique_geometries', metric='cardinality'))
group_hash = Quantity(
type=str,
description='Hashes that describe unique geometries simulated by this code run.',
- a_search=SearchQuantity(many_or='append', group='groups_grouped', metric_name='groups', metric='cardinality'))
+ a_search=Search(many_or='append', group='groups_grouped', metric_name='groups', metric='cardinality'))
labels = SubSection(
sub_section=Label, repeats=True,
@@ -170,7 +171,7 @@ class DFTMetadata(MSection):
a_search='labels')
optimade = SubSection(
- sub_section=optimade.OptimadeEntry,
+ sub_section=OptimadeEntry,
description='Metadata used for the optimade API.',
a_search='optimade')
@@ -182,7 +183,7 @@ class DFTMetadata(MSection):
if 'optimade' in kwargs:
print('########################## B')
- self.optimade = optimade.OptimadeEntry.m_from_dict(kwargs.pop('optimade'))
+ self.optimade = OptimadeEntry.m_from_dict(kwargs.pop('optimade'))
super().m_update(**kwargs)
@@ -293,4 +294,4 @@ class DFTMetadata(MSection):
self.labels.append(Label(label=aflow_id, type='prototype_id', source='aflow_prototype_library'))
# optimade
- self.optimade = backend.get_mi2_section(optimade.OptimadeEntry.m_def)
+ self.optimade = backend.get_mi2_section(OptimadeEntry.m_def)
diff --git a/nomad/datamodel/ems.py b/nomad/datamodel/ems.py
index 897b16a6f883b984597120f79b22d397d701a204..bb8b3ebd020e3baecd843e2e5fbc96382a6bf4c3 100644
--- a/nomad/datamodel/ems.py
+++ b/nomad/datamodel/ems.py
@@ -18,37 +18,37 @@ Experimental material science specific metadata
from nomad import utils, config
from nomad.metainfo import Quantity, MSection, Section, Datetime
-from nomad.metainfo.search import SearchQuantity
+from nomad.metainfo.search_extension import Search
-from .base import get_optional_backend_value
+from .common import get_optional_backend_value
class EMSMetadata(MSection):
m_def = Section(a_domain='ems')
# sample quantities
- chemical = Quantity(type=str, default='not processed', a_search=SearchQuantity())
- sample_constituents = Quantity(type=str, default='not processed', a_search=SearchQuantity(default_statistic=True))
- sample_microstructure = Quantity(type=str, default='not processed', a_search=SearchQuantity(default_statistic=True))
+ chemical = Quantity(type=str, default='not processed', a_search=Search())
+ sample_constituents = Quantity(type=str, default='not processed', a_search=Search(default_statistic=True))
+ sample_microstructure = Quantity(type=str, default='not processed', a_search=Search(default_statistic=True))
# general metadata
- experiment_summary = Quantity(type=str, default='not processed', a_search=SearchQuantity())
- experiment_location = Quantity(type=str, default='not processed', a_search=SearchQuantity())
- experiment_time = Quantity(type=Datetime, a_search=SearchQuantity())
+ experiment_summary = Quantity(type=str, default='not processed', a_search=Search())
+ experiment_location = Quantity(type=str, default='not processed', a_search=Search())
+ experiment_time = Quantity(type=Datetime, a_search=Search())
# method
- method = Quantity(type=str, default='not processed', a_search=SearchQuantity(default_statistic=True))
- probing_method = Quantity(type=str, default='not processed', a_search=SearchQuantity(default_statistic=True))
+ method = Quantity(type=str, default='not processed', a_search=Search(default_statistic=True))
+ probing_method = Quantity(type=str, default='not processed', a_search=Search(default_statistic=True))
# data metadata
- repository_name = Quantity(type=str, default='not processed', a_search=SearchQuantity())
- repository_url = Quantity(type=str, default='not processed', a_search=SearchQuantity())
- entry_repository_url = Quantity(type=str, default='not processed', a_search=SearchQuantity())
- preview_url = Quantity(type=str, default='not processed', a_search=SearchQuantity())
+ repository_name = Quantity(type=str, default='not processed', a_search=Search())
+ repository_url = Quantity(type=str, default='not processed', a_search=Search())
+ entry_repository_url = Quantity(type=str, default='not processed', a_search=Search())
+ preview_url = Quantity(type=str, default='not processed', a_search=Search())
# TODO move
- quantities = Quantity(type=str, shape=['0..*'], default=[], a_search=SearchQuantity())
- group_hash = Quantity(type=str, a_search=SearchQuantity())
+ quantities = Quantity(type=str, shape=['0..*'], default=[], a_search=Search())
+ group_hash = Quantity(type=str, a_search=Search())
def apply_domain_metadata(self, backend):
entry = self.m_parent
diff --git a/nomad/metainfo/optimade.py b/nomad/datamodel/optimade.py
similarity index 94%
rename from nomad/metainfo/optimade.py
rename to nomad/datamodel/optimade.py
index 18d45824c96d1172c5bddfbad692d9ae5c8dac0d..d1d6fee28c83bab250d769ff5b333791b959f199 100644
--- a/nomad/metainfo/optimade.py
+++ b/nomad/datamodel/optimade.py
@@ -2,8 +2,8 @@ from ase.data import chemical_symbols
from elasticsearch_dsl import Keyword, Float, InnerDoc, Nested
import numpy as np
-from . import MSection, Section, Quantity, SubSection, MEnum, units
-from .search import SearchQuantity
+from nomad.metainfo import MSection, Section, Quantity, SubSection, MEnum, units
+from nomad.metainfo.search_extension import Search
# TODO move the module
@@ -103,7 +103,7 @@ class OptimadeEntry(MSection):
elements = Quantity(
type=MEnum(chemical_symbols), shape=['1..*'],
links=optimade_links('h.6.2.1'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=True),
description='''
Names of the different elements present in the structure.
@@ -112,7 +112,7 @@ class OptimadeEntry(MSection):
nelements = Quantity(
type=int,
links=optimade_links('h.6.2.2'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=True),
description='''
Number of different elements in the structure as an integer.
@@ -121,7 +121,7 @@ class OptimadeEntry(MSection):
elements_ratios = Quantity(
type=float, shape=['nelements'],
links=optimade_links('h.6.2.3'),
- a_search=SearchQuantity(es_mapping=Nested(ElementRatio), es_value=ElementRatio.from_structure_entry),
+ a_search=Search(mapping=Nested(ElementRatio), value=ElementRatio.from_structure_entry),
a_optimade=Optimade(query=True, entry=True),
description='''
Relative proportions of different elements in the structure.
@@ -130,7 +130,7 @@ class OptimadeEntry(MSection):
chemical_formula_descriptive = Quantity(
type=str,
links=optimade_links('h.6.2.4'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=True),
description='''
The chemical formula for a structure as a string in a form chosen by the API
@@ -140,7 +140,7 @@ class OptimadeEntry(MSection):
chemical_formula_reduced = Quantity(
type=str,
links=optimade_links('h.6.2.5'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=True),
description='''
The reduced chemical formula for a structure as a string with element symbols and
@@ -150,7 +150,7 @@ class OptimadeEntry(MSection):
chemical_formula_hill = Quantity(
type=str,
links=optimade_links('h.6.2.6'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=False),
description='''
The chemical formula for a structure in Hill form with element symbols followed by
@@ -160,7 +160,7 @@ class OptimadeEntry(MSection):
chemical_formula_anonymous = Quantity(
type=str,
links=optimade_links('h.6.2.7'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=True),
description='''
The anonymous formula is the chemical_formula_reduced, but where the elements are
@@ -172,7 +172,7 @@ class OptimadeEntry(MSection):
dimension_types = Quantity(
type=int, shape=[3],
links=optimade_links('h.6.2.8'),
- a_search=SearchQuantity(es_value=lambda a: sum(a.dimension_types)),
+ a_search=Search(value=lambda a: sum(a.dimension_types)),
a_optimade=Optimade(query=True, entry=True),
description='''
List of three integers. For each of the three directions indicated by the three lattice
@@ -202,7 +202,7 @@ class OptimadeEntry(MSection):
nsites = Quantity(
type=int,
links=optimade_links('h.6.2.11'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=True), description='''
An integer specifying the length of the cartesian_site_positions property.
''')
@@ -221,7 +221,7 @@ class OptimadeEntry(MSection):
structure_features = Quantity(
type=MEnum(['disorder', 'unknown_positions', 'assemblies']), shape=['1..*'],
links=optimade_links('h.6.2.15'),
- a_search=SearchQuantity(),
+ a_search=Search(),
a_optimade=Optimade(query=True, entry=True), description='''
A list of strings that flag which special features are used by the structure.
diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py
index 11aa8d2d6f9ff50813684d0121df6499c56ead2d..4ac920486b5e146cf88a02e980f69c8e4adafc2c 100644
--- a/nomad/infrastructure.py
+++ b/nomad/infrastructure.py
@@ -96,9 +96,9 @@ def setup_elastic():
logger.info('setup elastic connection')
try:
- from nomad.search import Entry
- Entry.init(index=config.elastic.index_name)
- Entry._index._name = config.elastic.index_name
+ from nomad.search import entry_document
+ entry_document.init(index=config.elastic.index_name)
+ entry_document._index._name = config.elastic.index_name
logger.info('initialized elastic index', index_name=config.elastic.index_name)
except RequestError as e:
if e.status_code == 400 and 'resource_already_exists_exception' in e.error:
@@ -411,9 +411,9 @@ def reset(remove: bool):
if not elastic_client:
setup_elastic()
elastic_client.indices.delete(index=config.elastic.index_name)
- from nomad.search import Entry
+ from nomad.search import entry_document
if not remove:
- Entry.init(index=config.elastic.index_name)
+ entry_document.init(index=config.elastic.index_name)
logger.info('elastic index resetted')
except Exception as e:
logger.error('exception resetting elastic', exc_info=e)
diff --git a/nomad/metainfo/API_CONCEPT.md b/nomad/metainfo/API_CONCEPT.md
deleted file mode 100644
index 9729d05618e055ad7d0d2feb38d491aebbce0fc2..0000000000000000000000000000000000000000
--- a/nomad/metainfo/API_CONCEPT.md
+++ /dev/null
@@ -1,354 +0,0 @@
-**! This is not yet aligned with the ideas of CONCEPT.md !**
-
-# A new metainfo schema, interface, and *file formats* support
-
-This is a design document (later documentation) for a re-implementation of nomad's old
-meta-info *system*. Here *system* refers to the set of software components necessary to
-define (meta-)data structures and types, CRUD data according to schema via an abstract
-interface, and manage data in various resources
-(diff. file formats, databases, web resources, etc)
-
-## Example use cases
-
-We'll try to explain/design the system through a serious of use cases first. The
-respective examples are centered around a hypothetical python library that works as
-the meta-info systems interface.
-
-### Scientists using nomad parsers
-
-Imagine a student that uses VASP to simulate a system and wants to plot the density of states.
-She should be able to access the respective energies with:
-
-```python
-import nomad
-nomad.parse('TiO3.xml').run.single_configuration_calculation.dos.dos_energies.values
-```
-
-If she does not know what exactely `dos_energies` refers to:
-```python
-my_calc = nomad.parse('TiO3.xml')
-my_calc.run.single_configuration_calculation.dos.dos_energies.definition
-```
-
-It should give you something like:
-```json
-{
- "description": "Array containing the set of discrete energy values for the density ...",
- "share": ["number_of_dos_values"],
- "type": "float",
- "units": "J"
-}
-```
-
-But the units should actually not be fixed:
-
-```python
-my_calc.run.system.atom_positions.convert(nomad.units.angstrom).values
-```
-
-Values can be regular python lists or np arrays:
-```python
-my_calc.run.system.atom_positions.np_array
-```
-
-In the cases above, `system` is a list of systems. Therefore, the lines should return
-a list of actual values (e.g. a list of position matrices/np_arrays). To access
-a particular system:
-```python
-my_calc.run.system[0].atom_positions.np_array
-```
-
-To create more complex dict structures:
-```python
-from nomad import x
-my_calc.run.system(atom_positions=x.np_array, atom_labels=x.values, lattice_vector=x.np_array)
-```
-Should return a list of dictionaries with `atom_position`, `atom_labels`, `lattice_vector` as keys.
-The `x` acts as a surrogate for the retrived meta-info objects and everything accessed
-on `x`, will be accessed on the actual values.
-
-You can also create recursive *GraphQL* like queries:
-```python
-my_calc.run.system(atom_labels=x.values, symmetry=x(spacegroup=x.value)))
-```
-
-Or if another syntax is prefered:
-```python
-my_calc.run.system({
- atom_labels: x.values,
- symmetry: {
- spacegroup: x.value,
- hall_number: x.value
- }
-})
-```
-
-### Working with uploaded data
-
-There needs to be support for various resources, e.g. resources on the web, like the
-nomad archive:
-```python
-nomad.archive(upload_id='hh41jh4l1e91821').run.system.atom_labels.values
-```
-
-This can also be used to extend queries for calculations with queries for certain
-data points:
-```python
-nomad.archive(user='me@email.org', upload_name='last_upload').run.system(
- atom_labels=x.values, atom_positions=x.convert(units.angstrom).np_array)
-```
-In this case, it will return a generator that hides any API pagination. This this is now
-not a single run, it would generate a list (runs) or lists (systems) of dicts
-(with labels, and positions).
-
-
-### A parser creating data
-The CoE NOMAD parsing infrastructure used the concept of a *backend* that acted as an
-abstract interface to enter new data to the nomad archive. We like to think in terms of
-*resources*, where a resource can represent various storage media
-(e.g. in-memory, hdf5 file, something remote, etc.). Lets assume that the parser gets
-such a *resource* from the infrastructure to populate it with new data. Lets call the
-*resource* `backend` for old times sake:
-
-```python
-run = backend.run()
-system = run.system()
-system.atom_labels.values = ['Ti', 'O', 'O']
-system.atom_positions.values = [a1, a2, a3]
-system.close()
-system = run.system()
-# ...
-resource.close()
-```
-
-This basically describes the write interface. Of course parsers should also allow to read
-back all the properties they entered so far:
-```python
-system.atom_labels.values = ['Ti', 'O', 'O']
-system.atom_labels.values
-```
-
-The old backend allowed to build arrays piece by piece. This could also be possible here:
-```python
-positions = system.atom_positions.create()
-positions.add(a1)
-positions.add(a2)
-```
-
-## Core concepts
-
-
-### Conventions
-
-When mapping the following concepts to python implementations, we use the prefix `m_` on
-all methods and attributes that might conflict with user given names for meta info
-definitions.
-
-### Resources
-
-A *resource* refers to anything that can be used to *hold* data. This can be basic
-in python memory, a JSON file, an HDF5 file, a search index, a mongodb, or a remote
-resource that is accessible via REST API. Respective *resource* classes and their objects
-are used to parameterize access to the data. For example, to access a JSON file a
-file path is required, to store something in mongodb a connection to mongo and a collection
-is necessary, to read from an API an endpoint and possible parameters have to be configured.
-
-Beyond parameterized data access, all resources offer the same interface to navigate,
-enter, or modify data. The only exception are readonly resources that do not allow
-to add or modify data.
-
-The creation of resources could be allowed directly from a `nomad` package to create a
-very simple interface:
-```python
-nomad.parse('vasp_out.xml')
-nomad.open('archive.json')
-nomad.open('archive.hdf5')
-nomad.connect('mongodb://localhost/db_name', 'calculations')
-nomad.connect('https://nomad.fairdi.eu/archive/3892r478323/23892347')
-```
-
-The various resource implementations should offer the same interface. Necessary methods are
-- `close(save: bool = True)` to close a open file or connection
-- `save()` to save changes
-
-### Data Objects
-
-When navigating the contents of a resource (e.g. via `nomad.open('archive.json').run.system.atom_labels`),
-we start from a resource object (`nomad.open('archive.json')`) and pass various *data objects* (`.run.system.atom_labels`).
-There are obviously different types of data objects, i.e. *sections* (like `run`, `system`) and
-*properties* (like `atom_labels`). Sections and properties have to offer different interfaces.
-Sections need to allow to access and create subsections and properties. Properties have to allow to
-access, set, or modify the stored data.
-
-Independent of the object type, all data objects should allow to navigate to the definition (e.g. `run.m_definition`, `run.system.atom_labels.definition`).
-
-Navigation uses the user defined names of meta-info definitions for sections and properties.
-
-Section object need to support:
-- access to subsection via subsection name
-- access of properties via property name
-- array access for repeatable sections
-- navigation to its containing section: `.m_def`
-- allow to create/(re-)open subsections via calling the subsection name as a method: `.system()`
-- close a section so that the underlying resource implementation can potentially remove the section from memory and write it to a database/.hdf5 file
-- the *GraphQL* like access methods with dictionary to specify multiple sub-sections
-
-Property objects
-- access to values, depending on the shape and desired representation: `.value`, `.values`, `.np_array`
-- set values: `.values = ['Ti', 'O']`
-- create arrays and add values: `.atom_positions().add(...)`
-- convert units: `.convert(nomad.units.angstrom).values` or `.convert(nomad.units.angstrom).values = ...`
-
-### References and Resource Sets
-
-The meta-info allows to define references as a special property type. In this case the
-values of a property are references to other sections. This can be sections in the same
-resource or even other resources (across files). From an interface perspective, references
-can be set like any other property:
-
-```python
-system = run.system()
-calc = run.single_configuration_calculation()
-calc.system.value = system
-```
-
-Within the underlying resource, references are represented as URLs and these can be
-`local` or `remote`. URL paths define the position of a section in a resource. Local
-URLs only contain the path, remote URLs also contain a part that identifies the resource.
-
-The local reference in the previous example would be `/run/system/0`, referring to the first
-system in the run.
-
-If we create two different resource:
-```python
-systems = nomad.open('systems.json')
-calcs = nomad.open('calc.json')
-system_1 systems.system()
-system_2 systems.system()
-calc = calcs.run().single_configuration_calculation()
-calc.system = system_2
-systems.close()
-calcs.close()
-```
-
-the reference in the `calc.json` would be `file://systems.json/system/1`.
-
-When accessing a resource, other resources will be accessed on demand, when ever a reference
-needs to be resolved. The library can keep track of all accessed (and therefore open)
-resources through a global resource set. In more involved use-cases, it might be desirable
-that users can control resource sets via python.
-
-### Schemas and Definitions
-
-#### Introduction to schemas and definitions
-
-A schema comprises a set of definitions that define rules which govern what data does
-adhere to the schema and what not. We also say that we validate data against a schema to
-check if the data follows all the rules. In this sense, a schema defines an unlimited
-set of possible data that can be expressed in this schema.
-
-The definitions a schema can possibly contain is also govern by rules and these rules are also
-defined in a schema and this schema would be the schema of the schema. To be even
-more confusing, a schema can be the schema of itself. Meaning we can use the same set of
-definitions to formally define the definitions themselves. But lets start with an
-informal definition of schema elements.
-
-The following *elements* or kinds of definitions are used in metainfo schemas.
-
-#### Elements
-We call all kinds of definitions and their parts elements. Element is an abstract concept,
-in the sense that you cannot define elements directly. The abstract definition for elements
-barely defines a set of properties that is then shared by other elements.
-
-These properties include:
-- the elements *name* in the sense of a python compatible identifier (only a-bA-Z0-9_ and conventionally in camel case)
-- a human readable description, potentially in markdown format
-- a list of tags
-- a reference to a *section* definition. This denotes that instances of this element definition can be contained
-in instances of the referenced (parent) section definition.
-
-#### Sections
-A section definition is a special element. Sections will be used to create
-hierarchical structures of data. A section can contain other section and a set of properties.
-A section definition has the following properties: name, description, (parent) section (as all element definitions have), plus
-- a boolean *abstract* that determine if this section definition can be instantiated
-- a boolean *repeats* that determines if instances of this section can appear multiple times in their respective parent section
-- a references to another section definition called *extends* that denotes that instance of this definition can
-contain instances of all the element definitions that state the extended section definition as their (parent) section.
-
-#### Propertys
-A property definition is a special element. A property can be contained in a
-section and a property has a value. The type of the property values is determined by its property
-definition. Property value can be scalar, vectors, matrices, or higher-dimensional matrices.
-Each possible dimension can contain *primitive values*. Possible primitive value types are
-numerical types (int, float, int32, ...), bool, str, or references to other sections. Types
-of references are defined by referencing the respective section definition.
-
-A property definition has all element properties (name, description, (parent) section, tags) and has the following properties
-- *type* the data type that determine the possible values that the various dimensions can contain
-- a *shape* that determines how many primitive values each dimension of the properties value can contain. This can be a fix integer,
-a *wildcard* like 1..n or 0..n, or a references to another property with one dimension with a single int.
-- *units* is a list of strings that determines the physical unit of the various dimensions.
-
-#### Packages
-Packages are special elements. Packages can contain section definitions
-and property definitions. They have an additional property:
-- *dependencies* that list references to those packages that have definitions which complete the definitions
-in this package.
-
-#### annotation
-- name, value
-
-
-If we use the python interface described by example above, we can use it to define the
-metainfo schema schema in itself. Be aware that this might break your brain:
-```python
-from nomad import metainfo as mi
-from nomad.metainfo import types, dimensions
-
-resource = mi.open('metainfo.json')
-metainfo = resource.package(
- name='metainfo',
- description='The nomad meta-info schema schema')
-
-element = metainfo.section(name='element', abstract=True)
-metainfo.property(name='name', type=str, section=element)
-metainfo.property(name='description', type=str, section=element)
-super_section = metainfo.property(name='section', section=element)
-
-package = metainfo.section(name='package', extends=element)
-metainfo.property(name='dependency', section=package, types=[types.reference(package)], shape=[dimensions.0_n])
-
-element.section = package
-
-section = metainfo.section(name='section', repeats=True, extends=element, section=package)
-
-super_section.types = [nomad.types.reference(section)]
-
-metainfo.property(name='abstract', type=bool, section=section)
-metainfo.property(name='repeats', type=bool, section=section)
-metainfo.property(name='extends', type=types.reference(section), shape=[dimensions.0_n], section=section)
-
-property = metainfo.section(name='property', section=package, repeats=True)
-metainfo.property(name=types, type=types.type], section=property)
-metainfo.property(name=shape, type=types.union(types.reference(property), int, types.dimension), shape=[dimensions.0_n], section=property)
-metainfo.property(name=units, type=str, shape=[dimensions.0_n])
-
-annotation = metainfo.section(name='annotation', section=element, repeats=True)
-metainfo.property(name='key', type=str, section=annotations)
-metainfo.property(name='value', type=str, section=annotations)
-```
-
-In a similar manner, you can use the metainfo python interface to create metainfo
-schema definitions:
-```python
-resource = mi.open('common.json')
-common = resource.package(name='common')
-
-run = common.section(name='run')
-system = common.section(name='system', section='run')
-number_of_atoms = common.property(name='number_of_atoms', section=system)
-common.property(name='atom_labels', types=[str], shape=[number_of_atoms], section=system)
-common.property(name='atom_positions', types=[])
-
-```
diff --git a/nomad/metainfo/CONCEPT.md b/nomad/metainfo/CONCEPT.md
deleted file mode 100644
index f99214f37c77f143e75a1e41dff2d5e1adad0e05..0000000000000000000000000000000000000000
--- a/nomad/metainfo/CONCEPT.md
+++ /dev/null
@@ -1,269 +0,0 @@
-# NOMAD MetaInfo
-
-## History
-
-The NOMAD MetaInfo was devised within the first NOMAD CoE; over 2000 quantities have
-been defined in this *old MetaInfo*. The experience with this system revealed the following drawbacks:
-
-- The Python libraries that allow to use the MetaInfo are non pythonic and incomplete.
-- The MetaInfo is only used for the archive, not for the encyclopedia and repository data.
-- There is no direct support to map MetaInfo definitions to DI technologies (databases, search indices, APIs).
-- There is no support for namespaces. MetaInfo names are cumbersome. This will not scale to expected levels of FAIRmat metadata.
-- MetaInfo packages are not version controlled. They are part of the same git and do not belong to the independently evolving parsers. This does not allow for "external" parser development and makes it hard to keep versions consistent.
-- The MetaInfo is defined in JSON. The syntax is inadequate, checks are not immediate.
-- Attempts to revise the MetaInfo have failed in the past.
-
-## Goals
-
-### Common language to define physics (meta-)data quantities and their relationships
-
-The *physics quantities* part includes
-- each quantity MAY have a physics *unit*
-- each quantity MUST have a *shape* that precisely define vectors, matrices, tensors, and their dimensions
-- each quantity MAY have a numpy dtype that allows to map physics data to numpy arrays
-
-The *relationship* parts entails:
-- hierarchies for quantity *values* (e.g. *sections*)
-- hierarchies for quantity *definition* (e.g. *categories*, former *abstract types*)
-- *derived* quantities that can be computed from other quantities
-- *synonyms* as a special trivial case for derived quantities
-- *shapes* might also define a type of relationship through one quantity being the dimension of another
-- *references* between sections, via quantities that have a section definition as type
-
-In addition there are the *typical* data-type definition (schema, ontology, ...) features:
-- names/namespaces
-- modularization (i.e. Metainfo packages)
-- extensions: section inheritance, sections that add to other sections after definition
-- documentation
-- basic primitive types (int, string, bool)
-- simple compound types (lists, dictionaries, unions)
-- MAYBE an event mechanism
-
-### Complex, evolving, extendable packages of quantities
-
-There are a lot of quantities, and they need to be organized. There are three mechanisms
-to organize quantities:
-- *Packages* (a.k.a modules) allow to modularize large sets of quantities, e.g. one package per code
-- *Sections* allow to organize quantity values into containment (a.k.a whole-part, parent-child) hierarchies, e.g. `system` *contains* all quantity values that describe the simulated system.
-- *Categories* allow to organize quantity definitions via generalization (a.k.a specialization, inheritance) relationships, e.g. `atom_labels` and `formula_hill` (*special*) both express `chemical_composition` (*general*)
-
-Quantities and their relationships change over time. This requires (at least) a versioning mechanism to track changes and reason whether a pieces of data adheres to a certain version of the MetaInfo or not.
-
-The MetaInfo needs to be extendable. It must be possible to add *packages*, quantities in new *packages* must be addable to existing sections and categories. Existing sections must be extendable. It must be possible to develop and version packages independently.
-
-### Mappings to DI technologies
-
-The core of the MetaInfo is about defining data and their physics. But in the end, the data needs to be managed with DI components, such as file formats, databases, search indices, onotology tools, APIs, GUIs, programming languages, etc. While all these tools come with their own ways of defining data, it can be cumbersome to manually map the MetaInfo to the corresponding DI technology. Furthermore, this usually comprises both mapping definitions and transforming values.
-
-The MetaInfo will allow for quantity *annotations*. Annotations allow to add additional
-information to quantity definitions that carry the necessary information to automatically map/transform definitions and their values to underlying DI components. Annotations can be easily stripped/filtered to present the MetaInfo either clean or under technology specific lenses.
-
-### Intuitive programming interface to create, access, and use (meta-)data defined with the NOMAD MetaInfo
-
-While MetaInfo definitions and MetaInfo values should have a *native* serialization format (JSON), the primary interface to deal with definitions and data should be made from programming language (Python) primitives. By the way, both things are basically just mappings from the logical MetaInfo into concrete technologies (i.e. JSON, Python).
-
-As a programming language, Python has a far richer set of syntax to define and use data
-than JSON has. We should use this. It was not used for definitions in the NOMAD CoE, and
-the *backend*s for data were designed for creating data only and not very *pythonic*.
-
-## Concepts for a new NOMAD MetaInfo
-
-
-### Definition
-
-`Definition` is the abstract base for all definitions in the MetaInfo.
-
-- `name`, a string
-- `description`, a string
-- `links`, a list of URLs
-- `categories`, a list of references to category definitions
-- `annotations`, a list of `Annotations`
-
-- *derived*: `qualified_name`
-
-
-### Property
-
-`Property` is a special `Definition` and an abstract base for section properties.
-Properties define what data a section instance can hold. Properties are mapped to Python
-*descriptors*.
-
-- `section` specialized `parant` relation with the containing `Section`
-
-
-#### SubSections
-
-`SubSection` is a special `Property` that defines that a section instance can **contain**
-the instances of a sub section.
-
-- `sub_section` reference to the `Section` definition for the children
-- `repeats` is a boolean that determines if this sub section can be contain only once of multiple times
-
-- *constraint*: sub sections are not circular
-
-
-### Quantities (incl. dimensions, incl. references)
-
-A `Quantity` definition is a special and concrete `Property` definition:
-
-- `shape`, a list of either `int`, references to a dimension (quantity definition), or limits definitions (e.g. `'1..n'`, `'0..n'`.)
-- `type`, a primitive or MEnum type
-- `unit`, a (computed) units, e.g. `units.F * units.m`
-- `derived_from`, a list of references to other quantity definitions
-- `synonym`, a reference to another quantity definition
-
-*Dimensions* are quantity definitions with empty shape and int type.
-
-- *constraint*: `synonym`, `derived_from`, and dimensions come from the same section
-
-
-### Sections (incl. references)
-
-A `Section` is a special and concrete `Definition`.
-
-- `adds_to`, a reference to another section definition. All quantities of this *pseudo* section are added to the given section. (Might not be necessary)
-- `repeats`, a boolean
-- `extends`, list of reference to other section definitions. This section automatically inherits all quantities of the other sections. (Might not be necessary)
-
-- *derived*: `all_sub_sections`, all sub sections, included added and inherited ones, by name
-- *derived*: `all_quantities`, all quantities, included added and inherited ones, by name
-- *derived*: `all_properties`, all properties, included added and inherited ones, by name
-
-- *constraint*: `extends` is not circular
-- *constraint*: `adds_to` is not circular
-- *constraint*: all quantities that have *this* (or an `extends`) section as `section` have unique names
-
-`Section`s are mapped to Python classes/objects. `extends` is mapped to Python inheritance.
-
-
-### Categories
-
-A `Category` is a special `Definition`.
-
-- *constraint:* `Category` definition and its `categories` attribute do not form circles
-
-
-### Packages
-
-A `Package` is a special `Definition` that contains definitions. `Packages` are mapped
-to Python modules.
-
-- *derived*: `definitions`, all definitions in this package
-- *derived*: `sections`, all sections in this package
-- *derived*: `categories`, all categories in this package
-
-
-### Annotations
-
-Arbitrary serializable objects that can contain additional information.
-
-
-### MSection
-
-`MSection` is a Python base-class for all sections and provides additional reflection.
-
-- `m_def`: Python variable with the definition of this section
-- `m_data`: container for all the section data
-- `m_parent`: Python variable with the parent section instance
-- `m_parent_index`: Python variable with the index in the parent's repeatable sub section
-- `m_contents()`: all sub section instances
-- `m_all_contents()`: traverse all sub and sub sub section instances
-- `m_to_dict()`: serializable dict form
-- `m_to_json()`
-
-
-## Examples (of the Python interface)
-
-### Definitions
-
-This could be code, from a python module that represents the NOMAD *common* package `nomad.metainfo.common`:
-```python
-class System(MSection):
- '''
- The system is ...
- '''
-
- n_atoms = Quantity(type=int, derived_from='atom_labels')
-
- atom_labels = Quantity(
- shape=['n_atoms'],
- type=MEnum(ase.data.chemical_symbols),
- annotations=[ElasticSearchQuantity('keyword')])
- '''
- Atom labels are ...
- '''
-
- formula_hill = Quantity(type=str, derived_from=['atom_labels'])
-
- atom_species = Quantity(shape=['n_atoms'], type=int, derived_from='atom_labels')
-
- atom_positions = Quantity(shape=['n_atoms', 3], type=float, unit=units.m)
-
- cell = Quantity(shape=[3, 3], type=float, unit=units.m)
- lattice_vectors = Quantity(synonym='cell')
-
- pbc = Quantity(shape=[3], type=bool)
-
- # Not sure if this should be part of the definition. It will not serialize to
- # JSON. It might get complex for more involved cases. In many cases, we would
- # need both directions anyways. On the other hand, it allows to formally define
- # the derive semantics.
- def m_derive_atom_species(self) -> List[int]:
- return [ase.data.atomic_numbers[label] for label in self.atom_labels]
-
- def m_derive_n_atoms(self) -> int:
- return len(self.atom_labels)
-
-
-class Run(MSection):
-
- systems = SubSection(System, repeats=True)
-```
-
-This could be part of the VASP source code:
-```python
-class Method(MSection):
- m_definition = Section(adds_to=nomad.metainfo.common.Method)
-
- incar_nbands = Quantity(
- type=int, links=['https://cms.mpi.univie.ac.at/wiki/index.php/NBANDS'])
-```
-
-### (Meta-)data
-
-```python
-from nomad.metainfo.common import Run, System
-
-run = Run()
-
-system = run.systems.create(atom_labels=['H', 'H', 'O'])
-system.atom_positions = [[0, 0, 0], [1, 0, 0], [0.5, 0.5, 0]]
-system.cell = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]
-system.pbc = [False, False, False]
-
-print(system.atom_species) # [1, 1, 96]
-print(system.lattice_vectors)
-print(system.n_atoms)
-
-print(run.m_to_json(indent=2))
-```
-
-# Glossary
-
-A list of words with very specific and precise meaning. This meaning might not yet be
-fully expressed, but its there.
-
-- annotation
-- category
-- derived quantity
-- dimension
-- new MetaInfo
-- old MetaInfo
-- package
-- pythonic
-- quantity
-- reference
-- section
-- shape
-- synonym
-- unit
diff --git a/nomad/metainfo/README.md b/nomad/metainfo/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..5509b007298c291f728b28ce4c803e6b063651b0
--- /dev/null
+++ b/nomad/metainfo/README.md
@@ -0,0 +1,538 @@
+# NOMAD MetaInfo concept
+
+## History
+
+The NOMAD MetaInfo was devised within the first NOMAD CoE; over 2000 quantities have
+been defined in this *old MetaInfo*. The experience with this system revealed the following drawbacks:
+
+- The Python libraries that allow to use the MetaInfo are non pythonic and incomplete.
+- The MetaInfo is only used for the archive, not for the encyclopedia and repository data.
+- There is no direct support to map MetaInfo definitions to DI technologies (databases, search indices, APIs).
+- There is no support for namespaces. MetaInfo names are cumbersome. This will not scale to expected levels of FAIRmat metadata.
+- MetaInfo packages are not version controlled. They are part of the same git and do not belong to the independently evolving parsers. This does not allow for "external" parser development and makes it hard to keep versions consistent.
+- The MetaInfo is defined in JSON. The syntax is inadequate, checks are not immediate.
+- Attempts to revise the MetaInfo have failed in the past.
+
+## Goals
+
+### Common language to define physics (meta-)data quantities and their relationships
+
+The *physics quantities* part includes
+- each quantity MAY have a physics *unit*
+- each quantity MUST have a *shape* that precisely define vectors, matrices, tensors, and their dimensions
+- each quantity MAY have a numpy dtype that allows to map physics data to numpy arrays
+
+The *relationship* parts entails:
+- hierarchies for quantity *values* (e.g. *sections*)
+- hierarchies for quantity *definition* (e.g. *categories*, former *abstract types*)
+- *derived* quantities that can be computed from other quantities
+- *synonyms* as a special trivial case for derived quantities
+- *shapes* might also define a type of relationship through one quantity being the dimension of another
+- *references* between sections, via quantities that have a section definition as type
+
+In addition there are the *typical* data-type definition (schema, ontology, ...) features:
+- names/namespaces
+- modularization (i.e. Metainfo packages)
+- extensions: section inheritance, sections that add to other sections after definition
+- documentation
+- basic primitive types (int, string, bool)
+- simple compound types (lists, dictionaries, unions)
+- MAYBE an event mechanism
+
+### Complex, evolving, extendable packages of quantities
+
+There are a lot of quantities, and they need to be organized. There are three mechanisms
+to organize quantities:
+- *Packages* (a.k.a modules) allow to modularize large sets of quantities, e.g. one package per code
+- *Sections* allow to organize quantity values into containment (a.k.a whole-part, parent-child) hierarchies, e.g. `system` *contains* all quantity values that describe the simulated system.
+- *Categories* allow to organize quantity definitions via generalization (a.k.a specialization, inheritance) relationships, e.g. `atom_labels` and `formula_hill` (*special*) both express `chemical_composition` (*general*)
+
+Quantities and their relationships change over time. This requires (at least) a versioning mechanism to track changes and reason whether a pieces of data adheres to a certain version of the MetaInfo or not.
+
+The MetaInfo needs to be extendable. It must be possible to add *packages*, quantities in new *packages* must be addable to existing sections and categories. Existing sections must be extendable. It must be possible to develop and version packages independently.
+
+### Mappings to DI technologies
+
+The core of the MetaInfo is about defining data and their physics. But in the end, the data needs to be managed with DI components, such as file formats, databases, search indices, onotology tools, APIs, GUIs, programming languages, etc. While all these tools come with their own ways of defining data, it can be cumbersome to manually map the MetaInfo to the corresponding DI technology. Furthermore, this usually comprises both mapping definitions and transforming values.
+
+The MetaInfo will allow for quantity *annotations*. Annotations allow to add additional
+information to quantity definitions that carry the necessary information to automatically map/transform definitions and their values to underlying DI components. Annotations can be easily stripped/filtered to present the MetaInfo either clean or under technology specific lenses.
+
+### Intuitive programming interface to create, access, and use (meta-)data defined with the NOMAD MetaInfo
+
+While MetaInfo definitions and MetaInfo values should have a *native* serialization format (JSON), the primary interface to deal with definitions and data should be made from programming language (Python) primitives. By the way, both things are basically just mappings from the logical MetaInfo into concrete technologies (i.e. JSON, Python).
+
+As a programming language, Python has a far richer set of syntax to define and use data
+than JSON has. We should use this. It was not used for definitions in the NOMAD CoE, and
+the *backend*s for data were designed for creating data only and not very *pythonic*.
+
+## Concepts for a new NOMAD MetaInfo
+
+A schema comprises a set of definitions that define rules which govern what data does
+adhere to the schema and what not. We also say that we validate data against a schema to
+check if the data follows all the rules. In this sense, a schema defines an unlimited
+set of possible data that can be expressed in this schema.
+
+The definitions a schema can possibly contain is also govern by rules and these rules are also
+defined in a schema and this schema would be the schema of the schema. To be even
+more confusing, a schema can be the schema of itself. Meaning we can use the same set of
+definitions to formally define the definitions themselves. But lets start with an
+informal definition of schema elements.
+
+### Conventions
+
+When mapping the following concepts to python implementations, we use the prefix `m_` on
+all methods and attributes that might conflict with user given names for meta info
+definitions.
+
+### Definition
+We call all kinds of definitions and their parts definitions. Definition is an abstract concept,
+in the sense that you cannot define definitions directly. The abstract definition for elements
+barely defines a set of properties that is then shared by other elements.
+
+These properties include:
+- the elements *name* in the sense of a python compatible identifier (only a-bA-Z0-9_ and conventionally in camel case)
+- a human readable description, potentially in markdown format
+- a list of categories
+- annotations that attach possible metainfo extensions (db, search support, etc.) to the definition
+
+`Definition` is the abstract base for all definitions in the MetaInfo.
+
+- `name`, a string
+- `description`, a string
+- `links`, a list of URLs
+- `categories`, a list of references to category definitions
+- `annotations`, a list of `Annotations`
+
+- *derived*: `qualified_name`
+
+
+### Property
+
+`Property` is a special `Definition` and an abstract base for section properties.
+Properties define what data a section instance can hold. Properties are mapped to Python
+*descriptors*.
+
+- `section` specialized `parant` relation with the containing `Section`
+
+
+#### SubSections
+
+`SubSection` is a special `Property` that defines that a section instance can **contain**
+the instances of a sub section.
+
+- `sub_section` reference to the `Section` definition for the children
+- `repeats` is a boolean that determines if this sub section can be contain only once of multiple times
+
+- *constraint*: sub sections are not circular
+
+
+### Quantities (incl. dimensions, incl. references)
+
+A Quantity definition is a special definition. A quantity can be contained in a
+section and a quantity has a value. The type of the quantity values is determined by its quantity
+definition. Quantity value can be scalar, vectors, matrices, or higher-dimensional matrices.
+Each possible dimension can contain *primitive values*. Possible primitive value types are
+numerical types (int, float, int32, ...), bool, str, or references to other sections. Types
+of references are defined by referencing the respective section definition.
+
+A quantity definition has all definition quantities (name, description, ...) and has the following properties
+- *type* the data type that determine the possible values that the various dimensions can contain
+- a *shape* that determines how many primitive values each dimension of the quantities value can contain. This can be a fix integer,
+a *wildcard* like 1..n or 0..n, or a references to another property with one dimension with a single int.
+- *units* is a list of strings that determines the physical unit of the various dimensions.
+
+A `Quantity` definition is a special and concrete `Property` definition:
+
+- `shape`, a list of either `int`, references to a dimension (quantity definition), or limits definitions (e.g. `'1..n'`, `'0..n'`.)
+- `type`, a primitive or MEnum type
+- `unit`, a (computed) units, e.g. `units.F * units.m`
+- `derived_from`, a list of references to other quantity definitions
+- `synonym`, a reference to another quantity definition
+
+*Dimensions* are quantity definitions with empty shape and int type.
+
+- *constraint*: `synonym`, `derived_from`, and dimensions come from the same section
+
+
+### Sections (incl. references)
+
+A section definition is a special definition. Sections will be used to create
+hierarchical structures of data. A section can contain other section and a set of properties.
+A section definition has the following properties: name, description, (parent) section (as all element definitions have), plus
+- a boolean *abstract* that determine if this section definition can be instantiated
+- a boolean *repeats* that determines if instances of this section can appear multiple times in their respective parent section
+- a references to another section definition called *extends* that denotes that instance of this definition can
+contain instances of all the element definitions that state the extended section definition as their (parent) section.
+
+A `Section` is a special and concrete `Definition`.
+
+- `adds_to`, a reference to another section definition. All quantities of this *pseudo* section are added to the given section. (Might not be necessary)
+- `repeats`, a boolean
+- `extends`, list of reference to other section definitions. This section automatically inherits all quantities of the other sections. (Might not be necessary)
+
+- *derived*: `all_sub_sections`, all sub sections, included added and inherited ones, by name
+- *derived*: `all_quantities`, all quantities, included added and inherited ones, by name
+- *derived*: `all_properties`, all properties, included added and inherited ones, by name
+
+- *constraint*: `extends` is not circular
+- *constraint*: `adds_to` is not circular
+- *constraint*: all quantities that have *this* (or an `extends`) section as `section` have unique names
+
+`Section`s are mapped to Python classes/objects. `extends` is mapped to Python inheritance.
+
+
+### Categories
+
+A `Category` is a special `Definition`.
+
+- *constraint:* `Category` definition and its `categories` attribute do not form circles
+
+
+### Packages
+Packages are special definitions. Packages can contain section definitions
+and category definitions.
+
+A `Package` is a special `Definition` that contains definitions. `Packages` are mapped
+to Python modules.
+
+- *derived*: `definitions`, all definitions in this package
+- *derived*: `sections`, all sections in this package
+- *derived*: `categories`, all categories in this package
+
+
+### Annotations
+
+Arbitrary objects that can be attached to all definitions and contain additional information.
+
+
+### Resources
+
+A *resource* refers to anything that can be used to *hold* data. This can be basic
+in python memory, a JSON file, an HDF5 file, a search index, a mongodb, or a remote
+resource that is accessible via REST API. Respective *resource* classes and their objects
+are used to parameterize access to the data. For example, to access a JSON file a
+file path is required, to store something in mongodb a connection to mongo and a collection
+is necessary, to read from an API an endpoint and possible parameters have to be configured.
+
+Beyond parameterized data access, all resources offer the same interface to navigate,
+enter, or modify data. The only exception are readonly resources that do not allow
+to add or modify data.
+
+The creation of resources could be allowed directly from a `nomad` package to create a
+very simple interface:
+```python
+nomad.parse('vasp_out.xml')
+nomad.open('archive.json')
+nomad.open('archive.hdf5')
+nomad.connect('mongodb://localhost/db_name', 'calculations')
+nomad.connect('https://nomad.fairdi.eu/archive/3892r478323/23892347')
+```
+
+The various resource implementations should offer the same interface. Necessary methods are
+- `close(save: bool = True)` to close a open file or connection
+- `save()` to save changes
+
+### Data Objects
+
+When navigating the contents of a resource (e.g. via `nomad.open('archive.json').run.system.atom_labels`),
+we start from a resource object (`nomad.open('archive.json')`) and pass various *data objects* (`.run.system.atom_labels`).
+There are obviously different types of data objects, i.e. *sections* (like `run`, `system`) and
+*properties* (like `atom_labels`). Sections and properties have to offer different interfaces.
+Sections need to allow to access and create subsections and properties. Properties have to allow to
+access, set, or modify the stored data.
+
+Independent of the object type, all data objects should allow to navigate to the definition (e.g. `run.m_definition`, `run.system.atom_labels.definition`).
+
+Navigation uses the user defined names of meta-info definitions for sections and properties.
+
+Section object need to support:
+- access to subsection via subsection name
+- access of properties via property name
+- array access for repeatable sections
+- navigation to its containing section: `.m_def`
+- allow to create/(re-)open subsections via calling the subsection name as a method: `.system()`
+- close a section so that the underlying resource implementation can potentially remove the section from memory and write it to a database/.hdf5 file
+- the *GraphQL* like access methods with dictionary to specify multiple sub-sections
+
+Property objects
+- access to values, depending on the shape and desired representation: `.value`, `.values`, `.np_array`
+- set values: `.values = ['Ti', 'O']`
+- create arrays and add values: `.atom_positions().add(...)`
+- convert units: `.convert(nomad.units.angstrom).values` or `.convert(nomad.units.angstrom).values = ...`
+
+### References and Resource Sets
+
+The meta-info allows to define references as a special property type. In this case the
+values of a property are references to other sections. This can be sections in the same
+resource or even other resources (across files). From an interface perspective, references
+can be set like any other property:
+
+```python
+system = run.system()
+calc = run.single_configuration_calculation()
+calc.system.value = system
+```
+
+Within the underlying resource, references are represented as URLs and these can be
+`local` or `remote`. URL paths define the position of a section in a resource. Local
+URLs only contain the path, remote URLs also contain a part that identifies the resource.
+
+The local reference in the previous example would be `/run/system/0`, referring to the first
+system in the run.
+
+If we create two different resource:
+```python
+systems = nomad.open('systems.json')
+calcs = nomad.open('calc.json')
+system_1 systems.system()
+system_2 systems.system()
+calc = calcs.run().single_configuration_calculation()
+calc.system = system_2
+systems.close()
+calcs.close()
+```
+
+the reference in the `calc.json` would be `file://systems.json/system/1`.
+
+When accessing a resource, other resources will be accessed on demand, when ever a reference
+needs to be resolved. The library can keep track of all accessed (and therefore open)
+resources through a global resource set. In more involved use-cases, it might be desirable
+that users can control resource sets via python.
+
+### MSection
+
+`MSection` is a Python base-class for all sections and provides additional reflection.
+
+- `m_def`: Python variable with the definition of this section
+- `m_data`: container for all the section data
+- `m_parent`: Python variable with the parent section instance
+- `m_parent_index`: Python variable with the index in the parent's repeatable sub section
+- `m_contents()`: all sub section instances
+- `m_all_contents()`: traverse all sub and sub sub section instances
+- `m_to_dict()`: serializable dict form
+- `m_to_json()`
+
+
+## Examples (of the Python interface)
+
+### Definitions
+
+This could be code, from a python module that represents the NOMAD *common* package `nomad.metainfo.common`:
+```python
+class System(MSection):
+ '''
+ The system is ...
+ '''
+
+ n_atoms = Quantity(type=int, derived_from='atom_labels')
+
+ atom_labels = Quantity(
+ shape=['n_atoms'],
+ type=MEnum(ase.data.chemical_symbols),
+ annotations=[ElasticSearchQuantity('keyword')])
+ '''
+ Atom labels are ...
+ '''
+
+ formula_hill = Quantity(type=str, derived_from=['atom_labels'])
+
+ atom_species = Quantity(shape=['n_atoms'], type=int, derived_from='atom_labels')
+
+ atom_positions = Quantity(shape=['n_atoms', 3], type=float, unit=units.m)
+
+ cell = Quantity(shape=[3, 3], type=float, unit=units.m)
+ lattice_vectors = Quantity(synonym='cell')
+
+ pbc = Quantity(shape=[3], type=bool)
+
+ # Not sure if this should be part of the definition. It will not serialize to
+ # JSON. It might get complex for more involved cases. In many cases, we would
+ # need both directions anyways. On the other hand, it allows to formally define
+ # the derive semantics.
+ def m_derive_atom_species(self) -> List[int]:
+ return [ase.data.atomic_numbers[label] for label in self.atom_labels]
+
+ def m_derive_n_atoms(self) -> int:
+ return len(self.atom_labels)
+
+
+class Run(MSection):
+
+ systems = SubSection(System, repeats=True)
+```
+
+This could be part of the VASP source code:
+```python
+class Method(MSection):
+ m_definition = Section(adds_to=nomad.metainfo.common.Method)
+
+ incar_nbands = Quantity(
+ type=int, links=['https://cms.mpi.univie.ac.at/wiki/index.php/NBANDS'])
+```
+
+### (Meta-)data
+
+```python
+from nomad.metainfo.common import Run, System
+
+run = Run()
+
+system = run.systems.create(atom_labels=['H', 'H', 'O'])
+system.atom_positions = [[0, 0, 0], [1, 0, 0], [0.5, 0.5, 0]]
+system.cell = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]
+system.pbc = [False, False, False]
+
+print(system.atom_species) # [1, 1, 96]
+print(system.lattice_vectors)
+print(system.n_atoms)
+
+print(run.m_to_json(indent=2))
+```
+
+## Example use cases
+
+We'll try to explain/design the system through a serious of use cases first. The
+respective examples are centered around a hypothetical python library that works as
+the meta-info systems interface.
+
+### Scientists using nomad parsers
+
+Imagine a student that uses VASP to simulate a system and wants to plot the density of states.
+She should be able to access the respective energies with:
+
+```python
+import nomad
+nomad.parse('TiO3.xml').run.single_configuration_calculation.dos.dos_energies.values
+```
+
+If she does not know what exactely `dos_energies` refers to:
+```python
+my_calc = nomad.parse('TiO3.xml')
+my_calc.run.single_configuration_calculation.dos.dos_energies.definition
+```
+
+It should give you something like:
+```json
+{
+ "description": "Array containing the set of discrete energy values for the density ...",
+ "share": ["number_of_dos_values"],
+ "type": "float",
+ "units": "J"
+}
+```
+
+But the units should actually not be fixed:
+
+```python
+my_calc.run.system.atom_positions.convert(nomad.units.angstrom).values
+```
+
+Values can be regular python lists or np arrays:
+```python
+my_calc.run.system.atom_positions.np_array
+```
+
+In the cases above, `system` is a list of systems. Therefore, the lines should return
+a list of actual values (e.g. a list of position matrices/np_arrays). To access
+a particular system:
+```python
+my_calc.run.system[0].atom_positions.np_array
+```
+
+To create more complex dict structures:
+```python
+from nomad import x
+my_calc.run.system(atom_positions=x.np_array, atom_labels=x.values, lattice_vector=x.np_array)
+```
+Should return a list of dictionaries with `atom_position`, `atom_labels`, `lattice_vector` as keys.
+The `x` acts as a surrogate for the retrived meta-info objects and everything accessed
+on `x`, will be accessed on the actual values.
+
+You can also create recursive *GraphQL* like queries:
+```python
+my_calc.run.system(atom_labels=x.values, symmetry=x(spacegroup=x.value)))
+```
+
+Or if another syntax is prefered:
+```python
+my_calc.run.system({
+ atom_labels: x.values,
+ symmetry: {
+ spacegroup: x.value,
+ hall_number: x.value
+ }
+})
+```
+
+### Working with uploaded data
+
+There needs to be support for various resources, e.g. resources on the web, like the
+nomad archive:
+```python
+nomad.archive(upload_id='hh41jh4l1e91821').run.system.atom_labels.values
+```
+
+This can also be used to extend queries for calculations with queries for certain
+data points:
+```python
+nomad.archive(user='me@email.org', upload_name='last_upload').run.system(
+ atom_labels=x.values, atom_positions=x.convert(units.angstrom).np_array)
+```
+In this case, it will return a generator that hides any API pagination. This this is now
+not a single run, it would generate a list (runs) or lists (systems) of dicts
+(with labels, and positions).
+
+
+### A parser creating data
+The CoE NOMAD parsing infrastructure used the concept of a *backend* that acted as an
+abstract interface to enter new data to the nomad archive. We like to think in terms of
+*resources*, where a resource can represent various storage media
+(e.g. in-memory, hdf5 file, something remote, etc.). Lets assume that the parser gets
+such a *resource* from the infrastructure to populate it with new data. Lets call the
+*resource* `backend` for old times sake:
+
+```python
+run = backend.run()
+system = run.system()
+system.atom_labels.values = ['Ti', 'O', 'O']
+system.atom_positions.values = [a1, a2, a3]
+system.close()
+system = run.system()
+# ...
+resource.close()
+```
+
+This basically describes the write interface. Of course parsers should also allow to read
+back all the properties they entered so far:
+```python
+system.atom_labels.values = ['Ti', 'O', 'O']
+system.atom_labels.values
+```
+
+The old backend allowed to build arrays piece by piece. This could also be possible here:
+```python
+positions = system.atom_positions.create()
+positions.add(a1)
+positions.add(a2)
+```
+
+# Glossary
+
+A list of words with very specific and precise meaning. This meaning might not yet be
+fully expressed, but its there.
+
+- annotation
+- category
+- derived quantity
+- dimension
+- new MetaInfo
+- old MetaInfo
+- package
+- pythonic
+- quantity
+- reference
+- section
+- shape
+- synonym
+- unit
diff --git a/nomad/metainfo/__init__.py b/nomad/metainfo/__init__.py
index 0521e9d1cb0d621e00d45355459a6db6c56f0f1f..d95a7367492d6c23c66522a04d1097fdcad037f7 100644
--- a/nomad/metainfo/__init__.py
+++ b/nomad/metainfo/__init__.py
@@ -274,7 +274,29 @@ A more complex example
'''
-from .metainfo import MSection, MCategory, Definition, Property, Quantity, SubSection, \
- Section, Category, Package, Environment, MEnum, Datetime, MProxy, MetainfoError, DeriveError, \
- MetainfoReferenceError, DataType, MData, MDataDict, Reference, MResource, m_package, \
- units
+
+from .metainfo import (
+ MSection,
+ MCategory,
+ Definition,
+ Property,
+ Quantity,
+ SubSection,
+ Section,
+ Category,
+ Package,
+ Environment,
+ MEnum,
+ Datetime,
+ MProxy,
+ MetainfoError,
+ DeriveError,
+ MetainfoReferenceError,
+ DataType,
+ MData,
+ MDataDict,
+ Reference,
+ MResource,
+ m_package,
+ units,
+ Annotation)
diff --git a/nomad/metainfo/elastic.py b/nomad/metainfo/elastic.py
deleted file mode 100644
index c386703864b22cda1bf727d4b6cedeaedfad3d3f..0000000000000000000000000000000000000000
--- a/nomad/metainfo/elastic.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright 2018 Markus Scheidgen
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an"AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-'''
-Adds elastic search support to the metainfo.
-'''
-
-from . import Section, MSection
-
-
-def elastic_mapping(section: Section, base_cls: type) -> type:
- ''' Creates an elasticsearch_dsl document class from a section definition. '''
-
- dct = {
- name: quantity.m_annotations['elastic']['type']()
- for name, quantity in section.all_quantities.items()
- if 'elastic' in quantity.m_annotations}
-
- return type(section.name, (base_cls,), dct)
-
-
-def elastic_obj(source: MSection, target_cls: type):
- if source is None:
- return None
-
- assert isinstance(source, MSection), '%s must be an MSection decendant' % source.__class__.__name__
-
- target = target_cls()
-
- for name, quantity in source.m_def.all_quantities.items():
- elastic_annotation = quantity.m_annotations.get('elastic')
- if elastic_annotation is None:
- continue
-
- if 'mapping' in elastic_annotation:
- value = elastic_annotation['mapping'](source)
- else:
- value = getattr(source, name)
-
- setattr(target, name, value)
-
- return target
diff --git a/nomad/metainfo/elastic_extension.py b/nomad/metainfo/elastic_extension.py
new file mode 100644
index 0000000000000000000000000000000000000000..ee3d5fe8a0ac2090e6425de3591b2a5476eff112
--- /dev/null
+++ b/nomad/metainfo/elastic_extension.py
@@ -0,0 +1,244 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Any, Dict, cast
+import uuid
+
+
+from .metainfo import Section, Quantity, MSection, Annotation, MEnum, Datetime, Reference
+
+'''
+This module provides metainfo annotation class :class:`Elastic` and
+class:`ElasticDocument` that allows to configure metainfo definitions for the use of
+metainfo data in elastic search.
+'''
+
+
+class ElasticDocument(Annotation):
+ '''
+ This annotation class can be used to extend metainfo sections. It allows to detail
+ how section instances (and their sub sections and quantities) should be represented in
+ an elastic search index.
+
+ In general sections with this annotation are mapped to elasticsearch_dsl document
+ classes, sub sections become inner documents, and quantities with the :class:`Elastic`
+ extension become fields in their respective document.
+
+ Arguments:
+ index_name: This is used to optionally add the index_name to the resulting
+ elasticsearch_dsl document.
+ id: A callable that produces an id from a section instance that is used as id
+ for the respective elastic search index entry. The default will be randomly
+ generated UUID(4).
+
+ Attributes:
+ document: The elasticsearch_dsl document class that was generated from the
+ metainfo section
+ '''
+
+ _all_documents: Dict[str, Any] = {}
+
+ def __init__(self, index_name: str = None, id: Callable[[Any], str] = None):
+ self.index_name = index_name
+ self.id = id
+
+ self.m_def: Section = None
+ self.fields: Dict[Quantity, str] = {}
+
+ def init_annotation(self, definition):
+ assert isinstance(definition, Section), 'The ElasticDocument annotation is only usable with Sections.'
+ self.m_def = definition
+
+ @classmethod
+ def create_index_entry(cls, section: MSection):
+ ''' Creates an elasticsearch_dsl document instance for the given section. '''
+ m_def = section.m_def
+ annotation = m_def.m_x(ElasticDocument)
+ document_cls = ElasticDocument._all_documents[m_def.qualified_name()]
+
+ if annotation is None:
+ obj = document_cls()
+ else:
+ if annotation.id is not None:
+ id = annotation.id(section)
+ else:
+ id = uuid.uuid4()
+ obj = document_cls(meta=dict(id=id))
+
+ for quantity in m_def.all_quantities.values():
+ for annotation in quantity.m_x(Elastic, as_list=True):
+ if annotation.mapping is None:
+ continue
+
+ value = annotation.value(section)
+ if value is None or value == []:
+ continue
+
+ quantity_type = quantity.type
+ if isinstance(quantity_type, Reference):
+ if quantity.is_scalar:
+ value = ElasticDocument.create_index_entry(cast(MSection, value))
+ else:
+ value = [ElasticDocument.create_index_entry(item) for item in value]
+
+ setattr(obj, annotation.field, value)
+
+ for sub_section in m_def.all_sub_sections.values():
+ try:
+ if sub_section.repeats:
+ mi_values = list(section.m_get_sub_sections(sub_section))
+ if len(mi_values) == 0:
+ continue
+ value = [ElasticDocument.create_index_entry(value) for value in mi_values]
+ else:
+ mi_value = section.m_get_sub_section(sub_section, -1)
+ if mi_value is None:
+ continue
+ value = ElasticDocument.create_index_entry(mi_value)
+
+ setattr(obj, sub_section.name, value)
+ except KeyError:
+ # the sub section definition has no elastic quantities and therefore not
+ # corresponding document class
+ pass
+
+ return obj
+
+ @classmethod
+ def index(cls, section: MSection, **kwargs):
+ ''' Adds the given section to its elastic search index. '''
+ entry = cls.create_index_entry(section)
+ entry.save(**kwargs)
+ return entry
+
+ @property
+ def document(self):
+ return ElasticDocument.create_document(self.m_def)
+
+ @classmethod
+ def create_document(
+ cls, section: Section, inner_doc: bool = False, attrs: Dict[str, Any] = None,
+ prefix: str = None):
+ '''
+ Create all elasticsearch_dsl mapping classes for the section and its sub sections.
+ '''
+ document = cls._all_documents.get(section.qualified_name())
+ if document is not None:
+ return document
+
+ from elasticsearch_dsl import Document, InnerDoc, Keyword, Date, Integer, Boolean, Object
+
+ if attrs is None:
+ attrs = {}
+
+ # create an field for each sub section
+ for sub_section in section.all_sub_sections.values():
+ inner_document = ElasticDocument.create_document(
+ sub_section.sub_section, inner_doc=True, prefix=sub_section.name)
+ if inner_document is not None:
+ # sub sections with no elastic quantities get a None document
+ attrs[sub_section.name] = Object(inner_document)
+
+ # create an field for each quantity
+ for quantity in section.all_quantities.values():
+ first = True
+ for annotation in quantity.m_x(Elastic, as_list=True):
+ if annotation.mapping is None and first:
+ kwargs = dict(index=annotation.index)
+ # find a mapping based on quantity type
+ if quantity.type == str:
+ annotation.mapping = Keyword(**kwargs)
+ elif quantity.type == int:
+ annotation.mapping = Integer(**kwargs)
+ elif quantity.type == bool:
+ annotation.mapping = Boolean(**kwargs)
+ elif quantity.type == Datetime:
+ annotation.mapping = Date(**kwargs)
+ elif isinstance(quantity.type, Reference):
+ inner_document = ElasticDocument.create_document(
+ quantity.type.target_section_def, inner_doc=True,
+ prefix=annotation.field)
+ annotation.mapping = Object(inner_document)
+ elif isinstance(quantity.type, MEnum):
+ annotation.mapping = Keyword(**kwargs)
+ else:
+ raise NotImplementedError(
+ 'Quantity type %s for quantity %s is not supported.' % (quantity.type, quantity))
+
+ assert first or annotation.mapping is None, 'Only the first Elastic annotation is mapped'
+
+ if first:
+ assert annotation.field not in attrs, 'Elastic fields must be unique'
+ attrs[annotation.field] = annotation.mapping
+ annotation.register(prefix, annotation.field)
+
+ first = False
+
+ if len(attrs) == 0:
+ # do not create a document/inner document class, if no elastic quantities are defined
+ return None
+
+ document = type(section.name, (InnerDoc if inner_doc else Document,), attrs)
+ cls._all_documents[section.qualified_name()] = document
+ return document
+
+
+class Elastic(Annotation):
+ '''
+ This annotation class can be used to extend metainfo quantities. It allows to detail
+ how this quantity should be represented in an elastic search index.
+
+ Arguments:
+ field:
+ The name of the field for this quantity in the elastic search index. The
+ default is the quantity name.
+ mapping: A valid elasticsearch_dsl mapping. Default is ``Keyword()``.
+ value:
+ A callable that is applied to the containering section to get a value for
+ this quantity when saving the section in the elastic search index. By default
+ this will be the serialized quantity value.
+ index:
+ A boolean that indicates if this quantity should be indexed or merely be
+ part of the elastic document ``_source`` without being indexed for search.
+ '''
+ def __init__(
+ self,
+ field: str = None,
+ mapping: Any = None,
+ value: Callable[[Any], Any] = None,
+ index: bool = True):
+
+ self.field = field
+ self.mapping = mapping
+ self.value = value
+ self.index = index
+
+ self.prefix = None
+ self.qualified_field = field
+
+ def init_annotation(self, definition):
+ super().init_annotation(definition)
+
+ assert isinstance(definition, Quantity), 'The Elastic annotation is only usable with Quantities.'
+ if self.field is None:
+ self.field = definition.name
+
+ if self.value is None:
+ self.value = lambda section: section.m_get(definition)
+
+ def register(self, prefix: str, field: str):
+ if prefix is None:
+ self.qualified_field = field
+ else:
+ self.qualified_field = '%s.%s' % (prefix, field)
diff --git a/nomad/metainfo/flask_restplus.py b/nomad/metainfo/flask_extension.py
similarity index 100%
rename from nomad/metainfo/flask_restplus.py
rename to nomad/metainfo/flask_extension.py
diff --git a/nomad/metainfo/metainfo.py b/nomad/metainfo/metainfo.py
index ab8f85f03ba2a5273b0cc9d9938e3f84ca8700d0..163ca1d3a07b592624ee5024821b06b2968c7e61 100644
--- a/nomad/metainfo/metainfo.py
+++ b/nomad/metainfo/metainfo.py
@@ -656,7 +656,7 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
MetainfoError('Section has not m_def.')
# get annotations from kwargs
- self.m_annotations: Dict[str, Any] = {}
+ self.m_annotations: Dict[Union[str, type], Any] = {}
rest = {}
for key, value in kwargs.items():
if key.startswith('a_'):
@@ -688,7 +688,6 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
m_def.section_cls = cls
# add base sections
- extended_base_section = None
if m_def.extends_base_section:
base_sections_count = len(cls.__bases__)
if base_sections_count == 0:
@@ -700,7 +699,7 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
'Section %s extend the base section, but has more than one base section' % m_def)
base_section_cls = cls.__bases__[0]
- extended_base_section = base_section = getattr(base_section_cls, 'm_def', None)
+ base_section = getattr(base_section_cls, 'm_def', None)
if base_section is None:
raise MetainfoError(
'The base section of %s is not a section class.' % m_def)
@@ -710,7 +709,9 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
setattr(base_section_cls, name, attr)
section_to_add_properties_to = base_section
+ base_sections = [base_section]
else:
+ base_sections: List[Section] = []
for base_cls in cls.__bases__:
if base_cls != MSection:
base_section = getattr(base_cls, 'm_def')
@@ -719,12 +720,13 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
'Section defining classes must have MSection or a decendant as '
'base classes.')
- base_sections = list(m_def.m_get(Section.base_sections))
base_sections.append(base_section)
- m_def.m_set(Section.base_sections, base_sections)
section_to_add_properties_to = m_def
+ m_def.m_set(Section.base_sections, base_sections)
+
+ # transfer names, descriptions, constraints, event_handlers
constraints: Set[str] = set()
event_handlers: Set[Callable] = set(m_def.event_handlers)
for name, attr in cls.__dict__.items():
@@ -742,8 +744,6 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
else:
raise NotImplementedError('Unknown property kind.')
- attr.__init_property__()
-
if inspect.isfunction(attr):
method_name = attr.__name__
@@ -803,17 +803,7 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
if prop.description is None:
prop.description = param.description
- # validate
- def validate(definition):
- errors = definition.m_all_validate()
- if len(errors) > 0:
- raise MetainfoError(
- '%s. The section definition %s violates %d more constraints' %
- (str(errors[0]).strip('.'), definition, len(errors) - 1))
-
- if extended_base_section is not None:
- validate(extended_base_section)
- validate(m_def)
+ m_def.__init_metainfo__()
def __check_np(self, quantity_def: 'Quantity', value: np.ndarray) -> np.ndarray:
# TODO this feels expensive, first check, then possible convert very often?
@@ -1361,9 +1351,45 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas
return cast(MSectionBound, context)
- def m_x(self, key: str, default=None):
- ''' Convinience method for get the annotation with name ``key``. '''
- return self.m_annotations.get(key, default)
+ def m_x(self, key: Union[str, type], default=None, as_list: bool = False):
+ '''
+ Convinience method for get annotations
+
+ Arguments:
+ key: Either the optional annoation name or an annotation class. In the first
+ case the annotation is returned, regardless of its type. In the second
+ case, all names and list for names are iterated and all annotations of the
+ given class are returned.
+ default: The default, if no annotation is found. None is the default default.
+ as_list: Returns a list, no matter how many annoations have been found.
+ '''
+ if isinstance(key, str):
+ value = self.m_annotations.get(key, default)
+ if as_list and not isinstance(value, (list, tuple)):
+ return [value]
+ else:
+ return value
+
+ elif isinstance(key, type):
+ result_list = []
+ for values in self.m_annotations.values():
+ if isinstance(values, (tuple, list)):
+ for value in values:
+ if isinstance(value, key):
+ result_list.append(value)
+ elif isinstance(values, key):
+ result_list.append(values)
+
+ result_list_len = len(result_list)
+ if not as_list:
+ if result_list_len == 1:
+ return result_list[0]
+ elif result_list_len == 0:
+ return default
+
+ return result_list
+
+ raise TypeError('Key must be str or annotation class.')
def __validate_shape(self, quantity_def: 'Quantity', value):
if quantity_def == Quantity.default:
@@ -1518,6 +1544,23 @@ class Definition(MSection):
definitions = Definition.__all_definitions.setdefault(cls, [])
definitions.append(self)
+ def __init_metainfo__(self):
+ '''
+ An initialization method that is called after the class context of the definition
+ has been initialized. For example it is called on all quantities of a section
+ class after the class was created. If metainfo definitions are created without
+ a class context, this method must be called manually on all definitions.
+ '''
+
+ # initialize annotations
+ for annotation in self.m_annotations.values():
+ if isinstance(annotation, (tuple, list)):
+ for single_annotation in annotation:
+ if isinstance(single_annotation, Annotation):
+ single_annotation.init_annotation(self)
+ if isinstance(annotation, Annotation):
+ annotation.init_annotation(self)
+
@classmethod
def all_definitions(cls: Type[MSectionBound]) -> Iterable[MSectionBound]:
''' Class method that returns all definitions of this class.
@@ -1546,10 +1589,8 @@ class Definition(MSection):
class Property(Definition):
-
- def __init_property__(self):
- ''' Is called during section initialisation to allow property initialisation '''
- pass
+ ''' A common base-class for section properties: sub sections and quantities. '''
+ pass
class Quantity(Property):
@@ -1664,7 +1705,8 @@ class Quantity(Property):
# TODO derived_from = Quantity(type=Quantity, shape=['0..*'])
- def __init_property__(self):
+ def __init_metainfo__(self):
+ super().__init_metainfo__()
if self.derived is not None:
self.virtual = True
@@ -1891,6 +1933,26 @@ class Section(Definition):
self.all_sub_sections_by_section: Dict['Section', List['SubSection']] = dict()
self.parent_section_sub_section_defs: List['SubSection'] = list()
+ def __init_metainfo__(self):
+ # initialize properties
+ for property in self.all_properties.values():
+ property.__init_metainfo__()
+
+ super().__init_metainfo__()
+
+ # validate
+ def validate(definition):
+ errors = definition.m_all_validate()
+ if len(errors) > 0:
+ raise MetainfoError(
+ '%s. The section definition %s violates %d more constraints' %
+ (str(errors[0]).strip('.'), definition, len(errors) - 1))
+
+ if self.extends_base_section:
+ validate(self.m_get(Section.base_sections)[0])
+ else:
+ validate(self)
+
def on_add_sub_section(self, sub_section_def, sub_section):
if sub_section_def == Section.quantities:
self.all_properties[sub_section.name] = sub_section
@@ -2112,3 +2174,13 @@ class Environment(MSection):
if isinstance(definition, Definition):
definitions = self.all_definitions_by_name.setdefault(definition.name, [])
definitions.append(definition)
+
+
+class Annotation:
+ ''' Base class for annotations. '''
+
+ def __init__(self):
+ self.definition: Definition = None
+
+ def init_annotation(self, definition: Definition):
+ self.definition = definition
diff --git a/nomad/metainfo/mongoengine.py b/nomad/metainfo/mongoengine_extension.py
similarity index 100%
rename from nomad/metainfo/mongoengine.py
rename to nomad/metainfo/mongoengine_extension.py
diff --git a/nomad/metainfo/search.py b/nomad/metainfo/search.py
deleted file mode 100644
index 45f1d317fa3859f3c973ff8ae9d2aea9174197b8..0000000000000000000000000000000000000000
--- a/nomad/metainfo/search.py
+++ /dev/null
@@ -1,116 +0,0 @@
-from typing import Callable, Any
-
-from nomad import metainfo
-
-
-# TODO multi, split are more flask related
-class SearchQuantity:
- '''
- A metainfo quantity annotation class that defines additional properties that determine
- how to search for the respective quantity. Only quantities that have this will
- be mapped to elastic search.
-
- Attributes:
- name: The name of this search quantity. Will be the name in the elastic index and
- the name for the search parameter. Default is the metainfo quantity name.
- many_or: Indicates that an 'or' (es terms) search is performed if many values are given.
- Otherwise an 'and' (es bool->should->match) is performed. Values are 'split' and
- 'append' to indicate how URL search parameters should be treated.
- many_and: Indicates that many values can be supplied for search. Values are 'split' and
- 'append' to indicate how URL search parameters should be treated.
- order_default: Indicates that this quantity is used to order search results
- if no other ordering was specificed.
- metric: Quantity can be used to build statistics. Statistics provide a metric
- value for each value of the quantity. E.g. number of datasets with a given atom label.
- This defines a metric based on this quantity. Values need to be a valid
- elastic search aggregation (e.g. sum, cardinality, etc.).
- metric_name: If this quantity is indicated to function as a metric, the metric
- needs a name. By default the quantities name is used.
- default_statistic: Indicates this quantity to be part of the default statistics.
- statistics_size:
- The maximum number of values in a statistic. Default is 10.
- group: Indicates that his quantity can be used to group results. The value will
- be the name of the group.
- es_quantity: The quantity in the elastic mapping that is used to search. This is
- especially useful if the quantity represents a inner document and only one
- quantity of this inner object is used. Default is the name of the quantity.
- es_mapping: A valid elasticsearch_dsl mapping. Default is ``Keyword()``.
- es_value: A callable that is applied to section to get a value for this quantity in the elastic index.
- derived: A callable that is applied to search parameter values before search.
- '''
-
- def __init__(
- self,
- name: str = None, description: str = None,
- many_and: str = None, many_or: str = None,
- order_default: bool = False,
- group: str = None, metric: str = None, metric_name: str = None,
- default_statistic: bool = False,
- statistic_size: int = 10,
- es_quantity: str = None,
- es_mapping: Any = None,
- es_value: Callable[[Any], Any] = None,
- derived: Callable[[Any], Any] = None):
-
- self.name = name
- self.description = description
- self.many_and = many_and
- self.many_or = many_or
- self.order_default = order_default
- self.group = group
- self.default_statistic = default_statistic
- self.metric = metric
- self.metric_name = metric_name
- self.statistic_size = statistic_size
- self.es_quantity = es_quantity
- self.es_mapping = es_mapping
- self.es_value = es_value
- self.derived = derived
-
- self.prefix: str = None
- self.qualified_name: str = None
-
- assert many_and is None or many_or is None, 'A search quantity can only be used for multi or many search'
- assert many_and in [None, 'split', 'append'], 'Only split and append are valid values'
- assert many_or in [None, 'split', 'append'], 'Only split and append are valid values'
-
- def configure(self, quantity: metainfo.Quantity, prefix: str = None):
- if self.name is None:
- self.name = quantity.name
-
- if self.description is None:
- self.description = quantity.description
-
- if prefix is not None:
- self.qualified_name = '%s.%s' % (prefix, self.name)
- if self.es_quantity is not None:
- self.es_quantity = '%s.%s' % (prefix, self.es_quantity)
- if self.metric_name is not None:
- self.metric_name = '%s.%s' % (prefix, self.metric_name)
- if self.group is not None:
- self.group = '%s.%s' % (prefix, self.group)
- else:
- self.qualified_name = self.name
-
- if self.es_quantity is None:
- self.es_quantity = self.qualified_name
- if self.metric_name is None and self.metric is not None:
- self.metric_name = self.qualified_name
-
- @property
- def argparse_action(self):
- if self.many_or is not None:
- return self.many_or
-
- if self.many_and is not None:
- return self.many_and
-
- return None
-
- @property
- def many(self):
- return self.many_and is not None or self.many_or is not None
-
-
-def init(section: metainfo.MSection):
- pass
diff --git a/nomad/metainfo/search_extension.py b/nomad/metainfo/search_extension.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2cccec9f4affbed25114dc5d2170be65327916d
--- /dev/null
+++ b/nomad/metainfo/search_extension.py
@@ -0,0 +1,168 @@
+# Copyright 2020 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Any, Dict, List
+
+from nomad.metainfo.elastic_extension import Elastic
+
+
+search_quantities: Dict[str, 'Search'] = {}
+''' All available search quantities by their full qualified name. '''
+
+metrics: Dict[str, 'Search'] = {}
+'''
+The available search metrics. Metrics are integer values given for each entry that can
+be used in statistics (aggregations), e.g. the sum of all total energy calculations or
+cardinality of all unique geometries. The key is the metric name.
+'''
+
+groups: Dict[str, 'Search'] = {}
+''' The available groupable quantities. The key is the group name. '''
+
+order_default_quantities: Dict[str, 'Search'] = {}
+''' The quantity for each domain (key) that is the default quantity to order search results by. '''
+
+default_statistics: Dict[str, List['Search']] = {}
+''' A list of default statistics for each domain (key) '''
+
+
+# TODO multi, split are more flask related
+class Search(Elastic):
+ '''
+ A metainfo quantity annotation class that defines additional properties that determine
+ how to search for the respective quantity. Only quantities that have this will
+ be mapped to elastic search. The annotation is an extension of :class:`Elastic` and
+ add nomad API specific search features like grouping, statistics, metrics, domains, etc.
+
+ Attributes:
+ name: The name of this search quantity. Will be the name in the elastic index and
+ the name for the search parameter. Default is the metainfo quantity name.
+ many_or: Indicates that an 'or' (es terms) search is performed if many values are given.
+ Otherwise an 'and' (es bool->should->match) is performed. Values are 'split' and
+ 'append' to indicate how URL search parameters should be treated.
+ many_and: Indicates that many values can be supplied for search. Values are 'split' and
+ 'append' to indicate how URL search parameters should be treated.
+ order_default: Indicates that this quantity is used to order search results
+ if no other ordering was specificed.
+ metric: Quantity can be used to build statistics. Statistics provide a metric
+ value for each value of the quantity. E.g. number of datasets with a given atom label.
+ This defines a metric based on this quantity. Values need to be a valid
+ elastic search aggregation (e.g. sum, cardinality, etc.).
+ metric_name: If this quantity is indicated to function as a metric, the metric
+ needs a name. By default the quantities name is used.
+ default_statistic: Indicates this quantity to be part of the default statistics.
+ statistics_size:
+ The maximum number of values in a statistic. Default is 10.
+ group: Indicates that his quantity can be used to group results. The value will
+ be the name of the group.
+ search_field: The qualified field in the elastic mapping that is used to search.
+ This might be different from the field that is used to store the value in
+ elastic search. This is especially useful if the field represents a inner
+ document and a subfield of this inner object should be used for search.
+ derived: A callable that is applied to search parameter values before search.
+ '''
+
+ def __init__(
+ self,
+ name: str = None, description: str = None,
+ many_and: str = None, many_or: str = None,
+ order_default: bool = False,
+ group: str = None, metric: str = None, metric_name: str = None,
+ default_statistic: bool = False,
+ statistic_size: int = 10,
+ derived: Callable[[Any], Any] = None,
+ search_field: str = None,
+ **kwargs):
+
+ super().__init__(field=None, **kwargs)
+
+ self.name = name
+ self.description = description
+ self.many_and = many_and
+ self.many_or = many_or
+ self.order_default = order_default
+ self.group = group
+ self.default_statistic = default_statistic
+ self.metric = metric
+ self.metric_name = metric_name
+ self.statistic_size = statistic_size
+ self.search_field = search_field
+
+ self.derived = derived
+
+ self.qualified_name: str = None
+
+ assert many_and is None or many_or is None, 'A search quantity can only be used for multi or many search'
+ assert many_and in [None, 'split', 'append'], 'Only split and append are valid values'
+ assert many_or in [None, 'split', 'append'], 'Only split and append are valid values'
+
+ def init_annotation(self, definition):
+ if self.name is None:
+ self.name = definition.name
+ assert self.name is not None
+
+ if self.description is None:
+ self.description = definition.description
+
+ super().init_annotation(definition)
+
+ def register(self, prefix, field):
+ # TODO support more deeply nested quantities
+ domain_or_all = self.definition.m_parent.m_x('domain', '__all__')
+
+ prefix_and_dot = prefix + '.' if prefix is not None else ''
+
+ self.qualified_name = prefix_and_dot + self.name
+ if self.search_field is not None:
+ self.search_field = prefix_and_dot + self.search_field
+ else:
+ self.search_field = self.qualified_name
+
+ assert self.qualified_name not in search_quantities, 'Search quantities must have a unique name: %s' % self.name
+ search_quantities[self.qualified_name] = self
+
+ if self.metric is not None:
+ if self.metric_name is None:
+ self.metric_name = self.qualified_name
+ else:
+ self.metric_name = prefix_and_dot + self.metric_name
+
+ assert self.metric_name not in metrics, 'Metric names must be unique: %s' % self.metric_name
+ metrics[self.metric_name] = self
+
+ if self.group is not None:
+ self.group = prefix_and_dot + self.group
+ assert self.group not in groups, 'Groups must be unique'
+ groups[self.group] = self
+
+ if self.default_statistic:
+ default_statistics.setdefault(domain_or_all, []).append(self)
+
+ if self.order_default:
+ assert order_default_quantities.get(domain_or_all) is None, 'Only one quantity can be the order default'
+ order_default_quantities[domain_or_all] = self
+
+ @property
+ def argparse_action(self):
+ if self.many_or is not None:
+ return self.many_or
+
+ if self.many_and is not None:
+ return self.many_and
+
+ return None
+
+ @property
+ def many(self):
+ return self.many_and is not None or self.many_or is not None
diff --git a/nomad/normalizing/data/.gitignore b/nomad/normalizing/data/.gitignore
index 365491f487f16c60d20c5f292dfcea0ff73d9711..753400d2b6ef6154a2465ba04e6021aec7835c50 100644
--- a/nomad/normalizing/data/.gitignore
+++ b/nomad/normalizing/data/.gitignore
@@ -1 +1,2 @@
SM_all08.db
+springer.msg
\ No newline at end of file
diff --git a/nomad/normalizing/optimade.py b/nomad/normalizing/optimade.py
index d0ddf643ac1f4e497cc831d48645c4fcec926314..30bd8735b00389407d946d651e764b7a0c48357b 100644
--- a/nomad/normalizing/optimade.py
+++ b/nomad/normalizing/optimade.py
@@ -21,7 +21,7 @@ import pint.quantity
from nomad.normalizing.normalizer import SystemBasedNormalizer
from nomad.metainfo import units
-from nomad.metainfo.optimade import OptimadeEntry, Species
+from nomad.datamodel import OptimadeEntry, Species
species_re = re.compile(r'^([A-Z][a-z]?)(\d*)$')
diff --git a/nomad/processing/data.py b/nomad/processing/data.py
index 7a58323da1798476f82a242b6a4e3a8f450f7afd..af114814c72395fd002d19a6732627ad6df77d3f 100644
--- a/nomad/processing/data.py
+++ b/nomad/processing/data.py
@@ -303,7 +303,7 @@ class Calc(Proc):
entry_metadata.processed = False
self.metadata = entry_metadata.m_to_dict(include_defaults=True)
- search.create_entry(entry_metadata).save()
+ datamodel.EntryMetadata.m_def.m_x('elastic').index(entry_metadata)
except Exception as e:
self.get_logger().error('could not index after processing failure', exc_info=e)
@@ -432,7 +432,7 @@ class Calc(Proc):
# index in search
with utils.timer(logger, 'indexed', step='index'):
- search.create_entry(entry_metadata).save()
+ datamodel.EntryMetadata.m_def.m_x('elastic').index(entry_metadata)
# persist the archive
with utils.timer(
diff --git a/nomad/search.py b/nomad/search.py
index d30268fe1ecd287a57f136eca67dc716029bdd67..2fd6de16e236a139752a110262bcd7f6b334731a 100644
--- a/nomad/search.py
+++ b/nomad/search.py
@@ -16,16 +16,16 @@
This module represents calculations in elastic search.
'''
-from typing import Iterable, Dict, List, Any, Union, cast
-from elasticsearch_dsl import Document, InnerDoc, Keyword, Date, \
- Object, Boolean, Integer, Search, Q, A, analyzer, tokenizer
+from typing import Iterable, Dict, List, Any
+from elasticsearch_dsl import Search, Q, A, analyzer, tokenizer
import elasticsearch.helpers
from elasticsearch.exceptions import NotFoundError
from datetime import datetime
import json
-from nomad import config, datamodel, infrastructure, datamodel, utils, metainfo, processing as proc
-from nomad.metainfo.search import SearchQuantity
+from nomad import config, datamodel, infrastructure, datamodel, utils, processing as proc
+from nomad.metainfo.search_extension import search_quantities, metrics, order_default_quantities, default_statistics
+from nomad.metainfo.elastic_extension import ElasticDocument
path_analyzer = analyzer(
@@ -42,366 +42,22 @@ class ElasticSearchError(Exception): pass
class ScrollIdNotFound(Exception): pass
-_elastic_documents: Dict[str, Union[Document, InnerDoc]] = {}
-
-search_quantities: Dict[str, SearchQuantity] = {}
-''' All available search quantities by their full qualified name. '''
-
-metrics: Dict[str, SearchQuantity] = {}
-'''
-The available search metrics. Metrics are integer values given for each entry that can
-be used in statistics (aggregations), e.g. the sum of all total energy calculations or cardinality of
-all unique geometries.
-'''
-
-groups: Dict[str, SearchQuantity] = {}
-''' The available groupable quantities '''
-
-order_default_quantities: Dict[str, SearchQuantity] = {}
-
-default_statistics: Dict[str, List[SearchQuantity]] = {}
-
-
-# TODO make search the search quantities are initialized even without/before creating an elastic document
-# otherwise a dependency on import order is created
-def create_elastic_document(
- section: metainfo.Section, document_name: str = None, super_cls=Document,
- prefix: str = None, domain: str = None,
- attrs: Dict[str, Any] = None) -> Union[Document, InnerDoc]:
- '''
- Create all elasticsearch_dsl mapping classes for the section and its sub sections.
- '''
- domain = section.m_x('domain', domain)
- domain_or_all = domain if domain is not None else '__all__'
-
- if document_name is None:
- document_name = section.name
-
- if attrs is None:
- attrs = {}
-
- def get_inner_document(section: metainfo.Section, **kwargs) -> type:
- inner_document = _elastic_documents.get(section.qualified_name())
- if inner_document is None:
- inner_document = create_elastic_document(
- section, super_cls=InnerDoc, **kwargs)
-
- return inner_document
-
- # create an attribute for each sub section
- for sub_section in section.all_sub_sections.values():
- sub_section_prefix = sub_section.m_x('search')
- if sub_section_prefix is None:
- continue
-
- if prefix is not None:
- sub_section_prefix = '%s.%s' % (prefix, sub_section_prefix)
-
- inner_document = get_inner_document(
- sub_section.sub_section, domain=domain, prefix=sub_section_prefix)
- attrs[sub_section.name] = Object(inner_document)
-
- # create an attribute for each quantity
- for quantity in section.all_quantities.values():
- local_search_quantities = quantity.m_x('search')
-
- if local_search_quantities is None:
- continue
-
- if not isinstance(local_search_quantities, List):
- local_search_quantities = [local_search_quantities]
-
- for i, search_quantity in enumerate(local_search_quantities):
- search_quantity.configure(quantity=quantity, prefix=prefix)
-
- # only prefixed or top-level quantities are considered for being
- # searched directly. Other nested quantities can only be used via
- # other search_quantities's es_quantity
- if prefix is not None or super_cls == Document:
- qualified_name = search_quantity.qualified_name
- assert qualified_name not in search_quantities, 'Search quantities must have a unique name: %s' % qualified_name
- search_quantities[qualified_name] = search_quantity
-
- if search_quantity.metric is not None:
- qualified_metric_name = search_quantity.metric_name
- assert qualified_metric_name not in metrics, 'Metric names must be unique: %s' % qualified_metric_name
- metrics[qualified_metric_name] = search_quantity
-
- if search_quantity.group is not None:
- qualified_group = search_quantity.group
- assert qualified_group not in groups, 'Groups must be unique'
- groups[qualified_group] = search_quantity
-
- if search_quantity.default_statistic:
- default_statistics.setdefault(domain_or_all, []).append(search_quantity)
-
- if search_quantity.order_default:
- assert order_default_quantities.get(domain_or_all) is None, 'Only one quantity can be the order default'
- order_default_quantities[domain_or_all] = search_quantity
-
- if i != 0:
- # only the first quantity gets is mapped, unless the other has an
- # explicit mapping
- assert search_quantity.es_mapping is None, 'only the first quantity gets is mapped'
- continue
-
- if search_quantity.es_mapping is None:
- # find a mapping based on quantity type
- if quantity.type == str:
- search_quantity.es_mapping = Keyword()
- elif quantity.type == int:
- search_quantity.es_mapping = Integer()
- elif quantity.type == bool:
- search_quantity.es_mapping = Boolean()
- elif quantity.type == metainfo.Datetime:
- search_quantity.es_mapping = Date()
- elif isinstance(quantity.type, metainfo.Reference):
- inner_document = get_inner_document(quantity.type.target_section_def)
- search_quantity.es_mapping = Object(inner_document)
- elif isinstance(quantity.type, metainfo.MEnum):
- search_quantity.es_mapping = Keyword()
- else:
- raise NotImplementedError(
- 'Quantity type %s for quantity %s is not supported.' % (quantity.type, quantity))
-
- attrs[quantity.name] = search_quantity.es_mapping
-
- document = type(document_name, (super_cls,), attrs)
- _elastic_documents[section.qualified_name()] = document
- return document
-
-
-# TODO move to a init function that is triggered by elastic setup in infrastructure
-Entry = cast(Document, create_elastic_document(
- datamodel.EntryMetadata.m_def, document_name='Entry',
- attrs=dict(Index=type('Index', (), dict(name=config.elastic.index_name)))))
-''' The elasticsearch_dsl Document class that constitutes the entry index. '''
-
-metrics_names = list(metrics.keys())
-''' Names of all available metrics '''
+entry_document = datamodel.EntryMetadata.m_def.m_x(ElasticDocument).document
for domain in datamodel.domains:
order_default_quantities.setdefault(domain, order_default_quantities.get('__all__'))
- default_statistics.setdefault(domain, []).append(*default_statistics.get('__all__'))
-
-
-# class User(InnerDoc):
-
-# @classmethod
-# def from_user(cls, user):
-# self = cls(user_id=user.user_id)
-# self.name = user.name
-# self.email = user.email
-
-# return self
-
-# user_id = Keyword()
-# email = Keyword()
-# name = Text(fields={'keyword': Keyword()})
-
-
-# class Dataset(InnerDoc):
-
-# @classmethod
-# def from_dataset_id(cls, dataset_id):
-# dataset = datamodel.Dataset.m_def.m_x('me').get(dataset_id=dataset_id)
-# return cls(id=dataset.dataset_id, doi=dataset.doi, name=dataset.name, created=dataset.created)
-
-# id = Keyword()
-# doi = Keyword()
-# name = Keyword()
-# created = Date()
-
-
-# _domain_inner_doc_types: Dict[str, type] = {}
-
-
-# class WithDomain(IndexMeta):
-# ''' Override elasticsearch_dsl metaclass to sneak in domain specific mappings '''
-# def __new__(cls, name, bases, attrs):
-# for domain in Domain.instances.values():
-# inner_doc_type = _domain_inner_doc_types.get(domain.name)
-# if inner_doc_type is None:
-# domain_attrs = {
-# quantity.elastic_field: quantity.elastic_mapping
-# for quantity in domain.domain_quantities.values()}
-
-# inner_doc_type = type(domain.name, (InnerDoc,), domain_attrs)
-# _domain_inner_doc_types[domain.name] = inner_doc_type
-
-# attrs[domain.name] = Object(inner_doc_type)
-
-# return super(WithDomain, cls).__new__(cls, name, bases, attrs)
-
-
-# class Entry(Document, metaclass=WithDomain):
-
-# class Index:
-# name = config.elastic.index_name
-
-# domain = Keyword()
-# upload_id = Keyword()
-# upload_time = Date()
-# upload_name = Keyword()
-# calc_id = Keyword()
-# calc_hash = Keyword()
-# pid = Keyword()
-# raw_id = Keyword()
-# mainfile = Keyword()
-# files = Text(multi=True, analyzer=path_analyzer, fields={'keyword': Keyword()})
-# uploader = Object(User)
-
-# with_embargo = Boolean()
-# published = Boolean()
-
-# processed = Boolean()
-# last_processing = Date()
-# nomad_version = Keyword()
-# nomad_commit = Keyword()
-
-# authors = Object(User, multi=True)
-# owners = Object(User, multi=True)
-# comment = Text()
-# references = Keyword()
-# datasets = Object(Dataset)
-# external_id = Keyword()
-
-# atoms = Keyword()
-# only_atoms = Keyword()
-# formula = Keyword()
-
-# @classmethod
-# def from_entry_metadata(cls, source: datamodel.EntryMetadata) -> 'Entry':
-# entry = Entry(meta=dict(id=source.calc_id))
-# entry.update(source)
-# return entry
-
-# def update(self, source: datamodel.EntryMetadata) -> None:
-# self.domain = source.domain
-# self.upload_id = source.upload_id
-# self.upload_time = source.upload_time
-# self.upload_name = source.upload_name
-# self.calc_id = source.calc_id
-# self.calc_hash = source.calc_hash
-# self.pid = None if source.pid is None else str(source.pid)
-# self.raw_id = None if source.raw_id is None else str(source.raw_id)
-
-# self.processed = source.processed
-# self.last_processing = source.last_processing
-# self.nomad_version = source.nomad_version
-# self.nomad_commit = source.nomad_commit
-
-# self.mainfile = source.mainfile
-# if source.files is None:
-# self.files = [self.mainfile]
-# elif self.mainfile not in source.files:
-# self.files = [self.mainfile] + source.files
-# else:
-# self.files = source.files
-
-# self.with_embargo = bool(source.with_embargo)
-# self.published = source.published
-
-# uploader = datamodel.User.get(user_id=source.uploader) if source.uploader is not None else None
-# authors = [datamodel.User.get(user_id) for user_id in source.coauthors]
-# owners = [datamodel.User.get(user_id) for user_id in source.shared_with]
-# if uploader is not None:
-# authors.append(uploader)
-# owners.append(uploader)
-# authors.sort(key=lambda user: user.last_name + ' ' + user.first_name)
-# owners.sort(key=lambda user: user.last_name + ' ' + user.first_name)
-
-# self.uploader = User.from_user(uploader) if uploader is not None else None
-# self.authors = [User.from_user(user) for user in authors]
-# self.owners = [User.from_user(user) for user in owners]
-
-# self.comment = source.comment
-# self.references = source.references
-# self.datasets = [Dataset.from_dataset_id(dataset_id) for dataset_id in source.datasets]
-# self.external_id = source.external_id
-
-# self.atoms = source.atoms
-# self.only_atoms = nomad.datamodel.base.only_atoms(source.atoms)
-# self.formula = source.formula
-# self.n_atoms = source.n_atoms
-
-# if self.domain is not None:
-# inner_doc_type = _domain_inner_doc_types[self.domain]
-# inner_doc = inner_doc_type()
-# for quantity in Domain.instances[self.domain].domain_quantities.values():
-# quantity_value = quantity.elastic_value(getattr(source, quantity.metadata_field))
-# setattr(inner_doc, quantity.elastic_field, quantity_value)
-
-# setattr(self, self.domain, inner_doc)
-
-
-def create_entry(section: metainfo.MSection) -> Any:
- ''' Creates a elasticsearch_dsl document for the given section. '''
- cls = _elastic_documents[section.m_def.qualified_name()]
-
- if section.m_def == datamodel.EntryMetadata.m_def:
- obj = cls(meta=dict(id=section.m_get(datamodel.EntryMetadata.calc_id)))
- else:
- obj = cls()
-
- for quantity in section.m_def.all_quantities.values():
- search_quantities = quantity.m_x('search')
- if search_quantities is None:
- continue
-
- if not isinstance(search_quantities, list):
- search_quantities = [search_quantities]
-
- value = section.m_get(quantity)
- if value is None or value == []:
- continue
-
- for i, search_quantity in enumerate(search_quantities):
- if i != 0:
- # Only the value is only written for the first quantity
- continue
-
- quantity_type = quantity.type
- if isinstance(quantity_type, metainfo.Reference):
- if quantity.is_scalar:
- value = create_entry(cast(metainfo.MSection, value))
- else:
- value = [create_entry(item) for item in value]
-
- elif search_quantity.es_value is not None:
- value = search_quantity.es_value(section)
-
- setattr(obj, quantity.name, value)
-
- for sub_section in section.m_def.all_sub_sections.values():
- if not sub_section.m_x('search'):
- continue
-
- if sub_section.repeats:
- mi_values = list(section.m_get_sub_sections(sub_section))
- if len(mi_values) == 0:
- continue
- value = [create_entry(value) for value in mi_values]
- else:
- mi_value = section.m_get_sub_section(sub_section, -1)
- if mi_value is None:
- continue
- value = create_entry(mi_value)
-
- setattr(obj, sub_section.name, value)
-
- return obj
+ default_statistics.setdefault(domain, []).extend(default_statistics.get('__all__'))
def delete_upload(upload_id):
''' Delete all entries with given ``upload_id`` from the index. '''
- index = Entry._default_index()
+ index = entry_document._default_index()
Search(index=index).query('match', upload_id=upload_id).delete()
def delete_entry(calc_id):
''' Delete the entry with the given ``calc_id`` from the index. '''
- index = Entry._default_index()
+ index = entry_document._default_index()
Search(index=index).query('match', calc_id=calc_id).delete()
@@ -409,7 +65,7 @@ def publish(calcs: Iterable[datamodel.EntryMetadata]) -> None:
''' Update all given calcs with their metadata and set ``publish = True``. '''
def elastic_updates():
for calc in calcs:
- entry = create_entry(calc)
+ entry = calc.m_def.m_x('elastic').create_index_entry(calc)
entry.published = True
entry = entry.to_dict(include_meta=True)
source = entry.pop('_source')
@@ -430,7 +86,7 @@ def index_all(calcs: Iterable[datamodel.EntryMetadata], do_refresh=True) -> None
'''
def elastic_updates():
for calc in calcs:
- entry = create_entry(calc)
+ entry = calc.m_def.m_x('elastic').create_index_entry(calc)
entry = entry.to_dict(include_meta=True)
entry['_op_type'] = 'index'
yield entry
@@ -565,7 +221,7 @@ class SearchRequest:
value = [value]
if quantity.many_or and isinstance(value, List):
- self.q &= Q('terms', **{quantity.es_quantity: value})
+ self.q &= Q('terms', **{quantity.search_field: value})
return self
if quantity.derived:
@@ -579,7 +235,7 @@ class SearchRequest:
values = [value]
for item in values:
- self.q &= Q('match', **{quantity.es_quantity: item})
+ self.q &= Q('match', **{quantity.search_field: item})
return self
@@ -664,7 +320,7 @@ class SearchRequest:
The basic doc_count metric ``code_runs`` is always given.
'''
quantity = search_quantities[quantity_name]
- terms = A('terms', field=quantity.es_quantity, size=size, order=dict(_key='asc'))
+ terms = A('terms', field=quantity.search_field, size=size, order=dict(_key='asc'))
buckets = self._search.aggs.bucket('statistics:%s' % quantity_name, terms)
self._add_metrics(buckets, metrics_to_use)
@@ -677,7 +333,7 @@ class SearchRequest:
for metric in metrics_to_use:
metric_quantity = metrics[metric]
- field = metric_quantity.es_quantity
+ field = metric_quantity.search_field
parent.metric(
'metric:%s' % metric_quantity.metric_name,
A(metric_quantity.metric, field=field))
@@ -740,7 +396,7 @@ class SearchRequest:
size = 100
quantity = search_quantities[name]
- terms = A('terms', field=quantity.es_quantity)
+ terms = A('terms', field=quantity.search_field)
# We are using elastic searchs 'composite aggregations' here. We do not really
# compose aggregations, but only those pseudo composites allow us to use the
@@ -782,7 +438,9 @@ class SearchRequest:
Exectutes without returning actual results. Only makes sense if the request
was configured for statistics or quantity values.
'''
- return self._response(self._search.query(self.q)[0:0].execute())
+ search = self._search.query(self.q)[0:0]
+ response = search.execute()
+ return self._response(response)
def execute_scan(self, order_by: str = None, order: int = -1, **kwargs):
'''
@@ -795,9 +453,9 @@ class SearchRequest:
order_by_quantity = search_quantities[order_by]
if order == 1:
- search = search.sort(order_by_quantity.es_quantity)
+ search = search.sort(order_by_quantity.search_field)
else:
- search = search.sort('-%s' % order_by_quantity.es_quantity)
+ search = search.sort('-%s' % order_by_quantity.search_field)
search = search.params(preserve_order=True)
@@ -824,9 +482,9 @@ class SearchRequest:
search = self._search.query(self.q)
if order == 1:
- search = search.sort(order_by_quantity.es_quantity)
+ search = search.sort(order_by_quantity.search_field)
else:
- search = search.sort('-%s' % order_by_quantity.es_quantity)
+ search = search.sort('-%s' % order_by_quantity.search_field)
search = search[(page - 1) * per_page: page * per_page]
es_result = search.execute()
@@ -917,7 +575,7 @@ class SearchRequest:
# statistics
def get_metrics(bucket, code_runs):
result = {}
- for metric in metrics_names:
+ for metric in metrics:
agg_name = 'metric:%s' % metric
if agg_name in bucket:
result[metric] = bucket[agg_name]['value']
diff --git a/tests/app/test_api.py b/tests/app/test_api.py
index 1135267d414af571c399118286168d36e8edb1ca..5a3ed5580de03aa7f13090c507e275c8cab415ab 100644
--- a/tests/app/test_api.py
+++ b/tests/app/test_api.py
@@ -28,6 +28,7 @@ import itertools
from nomad.app.common import rfc3339DateTime
from nomad.app.api.auth import generate_upload_token
from nomad import search, parsing, files, config, utils, infrastructure
+from nomad.metainfo import search_extension
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
from nomad.datamodel import EntryMetadata, User, Dataset
@@ -716,7 +717,7 @@ class TestRepo():
entry_metadata.m_update(
calc_id='1', uploader=test_user.user_id, published=True, with_embargo=False)
- search.create_entry(entry_metadata).save(refresh=True)
+ EntryMetadata.m_def.m_x('elastic').index(entry_metadata, refresh=True)
entry_metadata.m_update(
calc_id='2', uploader=other_test_user.user_id, published=True,
@@ -725,17 +726,17 @@ class TestRepo():
entry_metadata.m_update(
atoms=['Fe'], comment='this is a specific word', formula='AAA')
entry_metadata.dft.basis_set = 'zzz'
- search.create_entry(entry_metadata).save(refresh=True)
+ EntryMetadata.m_def.m_x('elastic').index(entry_metadata, refresh=True)
entry_metadata.m_update(
calc_id='3', uploader=other_test_user.user_id, published=False,
with_embargo=False, pid=3, external_id='external_3')
- search.create_entry(entry_metadata).save(refresh=True)
+ EntryMetadata.m_def.m_x('elastic').index(entry_metadata, refresh=True)
entry_metadata.m_update(
calc_id='4', uploader=other_test_user.user_id, published=True,
with_embargo=True, pid=4, external_id='external_4')
- search.create_entry(entry_metadata).save(refresh=True)
+ EntryMetadata.m_def.m_x('elastic').index(entry_metadata, refresh=True)
yield
@@ -909,7 +910,7 @@ class TestRepo():
assert 'only_atoms' not in result
assert 'dft.basis_set' in result
- metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]
+ metrics_permutations = [[], search_extension.metrics] + [[metric] for metric in search_extension.metrics]
def test_search_admin(self, api, example_elastic_calcs, no_warn, admin_user_auth):
rv = api.get('/repo/?owner=admin', headers=admin_user_auth)
@@ -1797,8 +1798,7 @@ class TestDataset:
Calc(
calc_id='1', upload_id='1', create_time=datetime.datetime.now(),
metadata=entry_metadata.m_to_dict()).save()
- search.create_entry(entry_metadata).save()
- search.refresh()
+ EntryMetadata.m_def.m_x('elastic').index(entry_metadata, refresh=True)
def test_delete_dataset(self, api, test_user_auth, example_dataset_with_entry):
rv = api.delete('/datasets/ds1', headers=test_user_auth)
diff --git a/tests/conftest.py b/tests/conftest.py
index ff8713b2c870e656fcc40a3718e0f59e558b560f..3ea88cdf74eab657640a6e19a3cc1a915e8b315e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -34,7 +34,7 @@ import logging
from nomadcore.local_meta_info import loadJsonFile
import nomad_meta_info
-from nomad import config, infrastructure, parsing, processing, app, search, utils
+from nomad import config, infrastructure, parsing, processing, app, utils
from nomad.datamodel import User, EntryMetadata
from nomad.parsing import LocalBackend
@@ -698,7 +698,6 @@ def create_test_structure(
proc_calc = processing.Calc.from_entry_metadata(calc)
proc_calc.save()
- search_entry = search.create_entry(calc)
- search_entry.save()
+ calc.m_def.m_x('elastic').index(calc)
assert processing.Calc.objects(calc_id__in=[calc.calc_id]).count() == 1
diff --git a/tests/metainfo/test_elastic_extension.py b/tests/metainfo/test_elastic_extension.py
new file mode 100644
index 0000000000000000000000000000000000000000..641a59cb6c5a3c921c0b0cbd8b531157742b08d3
--- /dev/null
+++ b/tests/metainfo/test_elastic_extension.py
@@ -0,0 +1,78 @@
+# Copyright 2018 Markus Scheidgen
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an"AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from elasticsearch_dsl import Document
+
+from nomad.metainfo import MSection, Section, Quantity, SubSection, Reference
+from nomad.metainfo.elastic_extension import ElasticDocument, Elastic
+
+
+class User(MSection):
+
+ user_id = Quantity(type=str, a_elastic=Elastic())
+ name = Quantity(type=str, a_elastic=Elastic())
+ email = Quantity(type=str)
+
+
+class EMS(MSection):
+ experiment_method = Quantity(type=str)
+
+
+class DFT(MSection):
+ atoms = Quantity(
+ type=str, shape=['0..*'],
+ a_elastic=[Elastic(), Elastic()])
+ n_atoms = Quantity(
+ type=int, derived=lambda x: len(x.atoms),
+ a_eleastic=Elastic(index=False, field='natoms'))
+ only_atoms = Quantity(
+ type=str, shape=['0..*'], derived=lambda x: x.atoms,
+ a_elastic=Elastic(value=lambda x: ','.join(x.atoms)))
+
+
+class Entry(MSection):
+ m_def = Section(a_elastic=ElasticDocument(id=lambda entry: entry.entry_id))
+
+ entry_id = Quantity(type=str, a_elastic=Elastic())
+ uploader = Quantity(type=Reference(User.m_def), a_elastic=Elastic())
+
+ dft = SubSection(sub_section=DFT.m_def)
+ ems = SubSection(sub_section=EMS.m_def)
+
+
+def test_document():
+ annotation = Entry.m_def.m_x(ElasticDocument)
+ document = annotation.document
+
+ assert issubclass(document, Document)
+ assert DFT.m_def.qualified_name() in ElasticDocument._all_documents
+ assert EMS.m_def.qualified_name() not in ElasticDocument._all_documents
+
+ assert DFT.atoms.m_x(Elastic)[0].qualified_field == 'dft.atoms'
+ assert DFT.n_atoms.m_x(Elastic).qualified_field == 'dft.natoms'
+
+
+def test_create_entry():
+ user = User(user_id='test_user', name='Test Tester', email='test@testing.com')
+ entry = Entry(entry_id='test_id', uploader=user)
+ entry.m_create(DFT).atoms = ['H', 'O']
+
+ index_entry = Entry.m_def.m_x(ElasticDocument).create_index_entry(entry)
+
+ assert index_entry.entry_id == entry.entry_id
+ assert index_entry.uploader.name == 'Test Tester'
+ assert index_entry.dft.atoms == ['H', 'O']
+ assert index_entry.dft.natoms == 2
+ assert index_entry.dft.only_atoms == 'H,O'
+ assert hasattr(index_entry, 'ems') is False
diff --git a/tests/test_metainfo.py b/tests/metainfo/test_metainfo.py
similarity index 92%
rename from tests/test_metainfo.py
rename to tests/metainfo/test_metainfo.py
index abc98f6d0806fa391dd616ebec5bb7e31d711844..cd90acce6379397881ab521a089326f8be8df241 100644
--- a/tests/test_metainfo.py
+++ b/tests/metainfo/test_metainfo.py
@@ -19,8 +19,9 @@ import datetime
from nomadcore.local_meta_info import InfoKindEl, InfoKindEnv
-from nomad.metainfo.metainfo import MSection, MCategory, Section, Quantity, SubSection, \
- Definition, Package, DeriveError, MetainfoError, Environment, MResource, Datetime, units
+from nomad.metainfo.metainfo import (
+ MSection, MCategory, Section, Quantity, SubSection, Definition, Package, DeriveError,
+ MetainfoError, Environment, MResource, Datetime, units, Annotation)
from nomad.metainfo.example import Run, VaspRun, System, SystemHash, Parsing, m_package as example_package
from nomad.metainfo.legacy import LegacyMetainfoEnvironment
from nomad.parsing.metainfo import MetainfoBackend
@@ -235,6 +236,37 @@ class TestM2:
def test_derived_virtual(self):
assert System.n_atoms.virtual
+ def test_annotations(self):
+ class TestSectionAnnotation(Annotation):
+ def init_annotation(self, definition):
+ super().init_annotation(definition)
+ assert definition.name == 'TestSection'
+ assert 'test_quantity' in definition.all_quantities
+ assert definition.all_quantities['test_quantity'].m_x('test').initialized
+ assert definition.all_quantities['test_quantity'].m_x('test', as_list=True)[0].initialized
+ assert definition.all_quantities['test_quantity'].m_x(Annotation).initialized
+ assert all(a.initialized for a in definition.all_quantities['list_test_quantity'].m_x('test'))
+ assert all(a.initialized for a in definition.all_quantities['list_test_quantity'].m_x(Annotation))
+ self.initialized = True
+
+ class TestQuantityAnnotation(Annotation):
+ def init_annotation(self, definition):
+ super().init_annotation(definition)
+ assert definition.name in ['test_quantity', 'list_test_quantity']
+ assert definition.m_parent is not None
+ self.initialized = True
+
+ class TestSection(MSection):
+ m_def = Section(a_test=TestSectionAnnotation())
+
+ test_quantity = Quantity(type=str, a_test=TestQuantityAnnotation())
+ list_test_quantity = Quantity(
+ type=str,
+ a_test=[TestQuantityAnnotation(), TestQuantityAnnotation()])
+
+ assert TestSection.m_def.m_x('test').initialized
+ assert TestSection.m_def.m_x(TestSectionAnnotation).initialized
+
class TestM1:
''' Test for meta-info instances. '''
diff --git a/tests/test_datamodel.py b/tests/test_datamodel.py
index 2125b99d895cf60d334556763a0ae40ae168e056..0b2d6810f83c2cb35126a88092cfd378f3b2944c 100644
--- a/tests/test_datamodel.py
+++ b/tests/test_datamodel.py
@@ -116,7 +116,7 @@ if __name__ == '__main__':
import json
from elasticsearch.helpers import bulk
- from nomad import infrastructure, search
+ from nomad import infrastructure
print('Generate test data and add it to search and files')
print(' first arg is number of calcs (code runs)')
@@ -150,7 +150,7 @@ if __name__ == '__main__':
with upload_files.archive_log_file(calc.calc_id, 'wt') as f:
f.write('this is a generated test file')
- search_entry = search.Entry.from_entry_metadata(calc)
+ search_entry = calc.m_def.m_x('elastic').create_index_entry(calc)
search_entry.n_total_energies = random.choice(low_numbers_for_total_energies)
search_entry.n_geometries = low_numbers_for_geometries
for _ in range(0, random.choice(search_entry.n_geometries)):
diff --git a/tests/test_search.py b/tests/test_search.py
index cb8ef43873649b3274fe324da74a36484024f621..cfb33577d29265603349ed19ec282e7987310f08 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -17,7 +17,8 @@ from elasticsearch_dsl import Q
import pytest
from nomad import datamodel, search, processing, parsing, infrastructure, config
-from nomad.search import Entry, SearchRequest
+from nomad.search import entry_document, SearchRequest
+from nomad.metainfo import search_extension
def test_init_mapping(elastic):
@@ -144,11 +145,11 @@ def assert_metrics(container, metrics_names):
def test_search_statistics(elastic, example_search_data):
- assert 'authors' in search.metrics_names
- assert 'datasets' in search.metrics_names
- assert 'unique_entries' in search.metrics_names
+ assert 'authors' in search_extension.metrics.keys()
+ assert 'datasets' in search_extension.metrics.keys()
+ assert 'unique_entries' in search_extension.metrics.keys()
- use_metrics = search.metrics_names
+ use_metrics = search_extension.metrics.keys()
request = SearchRequest(domain='dft').statistic(
'dft.system', size=10, metrics_to_use=use_metrics).date_histogram()
@@ -166,7 +167,7 @@ def test_search_statistics(elastic, example_search_data):
def test_search_totals(elastic, example_search_data):
- use_metrics = search.metrics_names
+ use_metrics = search_extension.metrics.keys()
request = SearchRequest(domain='dft').totals(metrics_to_use=use_metrics)
results = request.execute()
@@ -232,7 +233,7 @@ def refresh_index():
def create_entry(entry_metadata: datamodel.EntryMetadata):
- entry = search.create_entry(entry_metadata)
+ entry = datamodel.EntryMetadata.m_def.m_x('elastic').index(entry_metadata)
entry.save()
assert_entry(entry_metadata.calc_id)
return entry
@@ -240,10 +241,10 @@ def create_entry(entry_metadata: datamodel.EntryMetadata):
def assert_entry(calc_id):
refresh_index()
- calc = Entry.get(calc_id)
+ calc = entry_document.get(calc_id)
assert calc is not None
- search = Entry.search().query(Q('term', calc_id=calc_id))[0:10]
+ search = entry_document.search().query(Q('term', calc_id=calc_id))[0:10]
assert search.count() == 1
results = list(hit.to_dict() for hit in search)
assert results[0]['calc_id'] == calc_id
@@ -254,7 +255,7 @@ def assert_search_upload(
additional_keys: List[str] = [], **kwargs):
keys = ['calc_id', 'upload_id', 'mainfile', 'calc_hash']
refresh_index()
- search_results = Entry.search().query('match_all')[0:10]
+ search_results = entry_document.search().query('match_all')[0:10]
assert search_results.count() == len(list(upload_entries))
if search_results.count() > 0:
for hit in search_results:
@@ -292,7 +293,7 @@ if __name__ == '__main__':
def gen_data():
for pid in range(0, n):
calc = generate_calc(pid)
- calc = Entry.from_entry_metadata(calc)
+ calc = entry_document.from_entry_metadata(calc)
yield calc.to_dict(include_meta=True)
bulk(infrastructure.elastic_client, gen_data())