diff --git a/nomad/app/api/info.py b/nomad/app/api/info.py
index b284ea9193d166860d7bec331b66a2c3de9692f6..6e7196c0f4135d86ecd34bdc65fcbbcff644270f 100644
--- a/nomad/app/api/info.py
+++ b/nomad/app/api/info.py
@@ -69,25 +69,30 @@ class InfoResource(Resource):
@api.marshal_with(info_model, skip_none=True, code=200, description='Info send')
def get(self):
""" Return information about the nomad backend and its configuration. """
+ codes = [
+ parser.code_name
+ for parser in parsing.parser_dict.values()
+ if isinstance(parser, parsing.MatchingParser) and parser.domain == 'dft']
+
return {
'parsers': [
key[key.index('/') + 1:]
for key in parsing.parser_dict.keys()],
- 'codes': sorted(set([
- parser.code_name
- for parser in parsing.parser_dict.values()
- if isinstance(parser, parsing.MatchingParser) and parser.domain == datamodel.Domain.instance.name]), key=lambda x: x.lower()),
+ 'codes': sorted(set(codes), key=lambda x: x.lower()),
'normalizers': [normalizer.__name__ for normalizer in normalizing.normalizers],
- 'domain': {
- 'name': datamodel.Domain.instance.name,
- 'quantities': [quantity for quantity in datamodel.Domain.instance.quantities.values()],
- 'metrics_names': datamodel.Domain.instance.metrics_names,
- 'aggregations_names': datamodel.Domain.instance.aggregations_names,
- 'metainfo': {
- 'all_package': datamodel.Domain.instance.metainfo_all_package,
- 'root_sections': datamodel.Domain.instance.root_sections
+ 'domains': [
+ {
+ 'name': domain.name,
+ 'quantities': [quantity for quantity in domain.quantities.values()],
+ 'metrics_names': domain.metrics_names,
+ 'aggregations_names': domain.aggregations_names,
+ 'metainfo': {
+ 'all_package': domain.metainfo_all_package,
+ 'root_sections': domain.root_sections
+ }
}
- },
+ for domain in datamodel.Domain.instances.values()
+ ],
'version': config.version,
'release': config.release,
'git': {
diff --git a/nomad/app/api/repo.py b/nomad/app/api/repo.py
index 3df3ce5576a5431fd6c98714bb1291d27a00a834..b5f5df82248c0dcb5b64dc541088fe6ac9799835 100644
--- a/nomad/app/api/repo.py
+++ b/nomad/app/api/repo.py
@@ -148,7 +148,8 @@ class RepoCalcsResource(Resource):
The search will return aggregations on a predefined set of quantities. Aggregations
will tell you what quantity values exist and how many entries match those values.
- Ordering is determined by ``order_by`` and ``order`` parameters.
+ Ordering is determined by ``order_by`` and ``order`` parameters. Default is
+ ``upload_time`` in decending order.
"""
try:
@@ -162,7 +163,7 @@ class RepoCalcsResource(Resource):
page = args.get('page', 1)
per_page = args.get('per_page', 10 if not scroll else 1000)
order = args.get('order', -1)
- order_by = args.get('order_by', 'formula')
+ order_by = args.get('order_by', 'upload_time')
date_histogram = args.get('date_histogram', False)
metrics: List[str] = request.args.getlist('metrics')
@@ -594,9 +595,8 @@ class RepoQuantityResource(Resource):
except AssertionError:
abort(400, message='invalid size')
- search_request.quantity(quantity, size=size, after=after)
-
try:
+ search_request.quantity(quantity, size=size, after=after)
results = search_request.execute()
quantities = results.pop('quantities')
results['quantity'] = quantities[quantity]
diff --git a/nomad/app/optimade/endpoints.py b/nomad/app/optimade/endpoints.py
index d41f16894db942cb80f8166659fc1564e71ce93a..9efb314e01825993b6037f159fbcccd1e508b0de 100644
--- a/nomad/app/optimade/endpoints.py
+++ b/nomad/app/optimade/endpoints.py
@@ -43,7 +43,7 @@ def base_request_args():
def base_search_request():
""" Creates a search request for all public and optimade enabled data. """
return search.SearchRequest().owner('all', None).query(
- Q('exists', field='optimade.nelements')) # TODO use the elastic annotations when done
+ Q('exists', field='dft.optimade.nelements')) # TODO use the elastic annotations when done
@ns.route('/calculations')
diff --git a/nomad/app/optimade/filterparser.py b/nomad/app/optimade/filterparser.py
index f3d9aa602b6da642e5c1b5f11b09cc060b89dfa4..ce3d3abffe38721eba49132f8429f3cbe7ca228c 100644
--- a/nomad/app/optimade/filterparser.py
+++ b/nomad/app/optimade/filterparser.py
@@ -26,7 +26,7 @@ class FilterException(Exception):
quantities: Dict[str, Quantity] = {
q.name: Quantity(
- q.name, es_field='optimade.%s' % q.name,
+ q.name, es_field='dft.optimade.%s' % q.name,
elastic_mapping_type=q.m_annotations['elastic']['type'])
for q in OptimadeEntry.m_def.all_quantities.values()
@@ -34,7 +34,7 @@ quantities: Dict[str, Quantity] = {
quantities['elements'].length_quantity = quantities['nelements']
quantities['dimension_types'].length_quantity = quantities['dimension_types']
-quantities['elements'].has_only_quantity = Quantity(name='only_atoms')
+quantities['elements'].has_only_quantity = Quantity(name='dft.only_atoms')
quantities['elements'].nested_quantity = quantities['elements_ratios']
quantities['elements_ratios'].nested_quantity = quantities['elements_ratios']
diff --git a/nomad/cli/admin/uploads.py b/nomad/cli/admin/uploads.py
index 0eb5186f8c38cfcf551f323711fbb050fcfa0ece..2671cb4c79f55b6e0aa95cd99b56a5d130ded2b7 100644
--- a/nomad/cli/admin/uploads.py
+++ b/nomad/cli/admin/uploads.py
@@ -50,7 +50,7 @@ def uploads(ctx, user: str, staging: bool, processing: bool, outdated: bool, cod
query &= Q(upload_id__in=uploads)
if code is not None and len(code) > 0:
- code_queries = [es.Q('match', code_name=code_name) for code_name in code]
+ code_queries = [es.Q('match', **{'dft.code_name': code_name}) for code_name in code]
code_query = es.Q('bool', should=code_queries, minimum_should_match=1)
code_search = es.Search(index=config.elastic.index_name)
diff --git a/nomad/cli/client/statistics.py b/nomad/cli/client/statistics.py
index fa3e30fdd4ce221c3109e3defce79757b1131cbc..4189e6210b3d8a81d2ee4e813424380c385dc4a5 100644
--- a/nomad/cli/client/statistics.py
+++ b/nomad/cli/client/statistics.py
@@ -34,7 +34,7 @@ def codes(client, minimum=1, **kwargs):
data = client.repo.search(per_page=1, **kwargs).response().result
x_values = sorted([
- code for code, values in data.statistics['code_name'].items()
+ code for code, values in data.statistics['dft.code_name'].items()
if code != 'not processed' and values.get('calculations', 1000) >= minimum], key=lambda x: x.lower())
return data.statistics, x_values, 'code_name', 'code'
@@ -61,7 +61,7 @@ def error_fig(client):
return {
code: values[metric]
- for code, values in result.quantities['code_name'].items()
+ for code, values in result.quantities['dft.code_name'].items()
if code != 'not processed' and (not labels or code in labels) > 0}
# get the data
@@ -414,7 +414,7 @@ def statistics_table(html, geometries, public_path):
'total', 'all', 'code_runs')
phonons = get_statistic(
- client.repo.search(per_page=1, code_name='Phonopy').response().result,
+ client.repo.search(per_page=1, **{'dft.code_name': 'Phonopy'}).response().result,
'total', 'all', 'code_runs')
# files and sized
diff --git a/nomad/datamodel/dft.py b/nomad/datamodel/dft.py
index 585916b4351ed9f698c7f8dd8d38213150a59400..8139b0fa2bac26fafeeff7bf6c7c06fad48065ea 100644
--- a/nomad/datamodel/dft.py
+++ b/nomad/datamodel/dft.py
@@ -335,4 +335,4 @@ Domain(
groups=dict(
groups=('group_hash', 'groups')),
default_statistics=[
- 'atoms', 'basis_set', 'xc_functional', 'system', 'crystal_system', 'code_name'])
+ 'dft.atoms', 'dft.basis_set', 'dft.xc_functional', 'dft.system', 'dft.crystal_system', 'dft.code_name'])
diff --git a/tests/app/test_api.py b/tests/app/test_api.py
index 8e7f6bee03a8e88c47443bb15de8d3656a8478d6..04b023df8107610b3a0f6f6e8dcce9d8861454d8 100644
--- a/tests/app/test_api.py
+++ b/tests/app/test_api.py
@@ -815,7 +815,7 @@ class TestRepo():
data = self.assert_search(rv, calcs)
results = data.get('results', None)
if calcs > 0:
- for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
+ for key in ['uploader', 'calc_id', 'dft.formula', 'upload_id']:
assert key in results[0]
@pytest.mark.parametrize('calcs, start, end', [
@@ -844,22 +844,22 @@ class TestRepo():
self.assert_search(rv, calcs)
@pytest.mark.parametrize('calcs, quantity, value, user', [
- (2, 'system', 'bulk', 'test_user'),
- (0, 'system', 'atom', 'test_user'),
- (1, 'atoms', 'Br', 'test_user'),
- (1, 'atoms', 'Fe', 'test_user'),
- (0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
- (0, 'only_atoms', ['Br', 'Si'], 'test_user'),
- (1, 'only_atoms', ['Fe'], 'test_user'),
- (1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
- (1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
+ (2, 'dft.system', 'bulk', 'test_user'),
+ (0, 'dft.system', 'atom', 'test_user'),
+ (1, 'dft.atoms', 'Br', 'test_user'),
+ (1, 'dft.atoms', 'Fe', 'test_user'),
+ (0, 'dft.atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
+ (0, 'dft.only_atoms', ['Br', 'Si'], 'test_user'),
+ (1, 'dft.only_atoms', ['Fe'], 'test_user'),
+ (1, 'dft.only_atoms', ['Br', 'K', 'Si'], 'test_user'),
+ (1, 'dft.only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(1, 'comment', 'specific', 'test_user'),
(1, 'authors', 'Leonard Hofstadter', 'test_user'),
(2, 'files', 'test/mainfile.txt', 'test_user'),
(2, 'paths', 'mainfile.txt', 'test_user'),
(2, 'paths', 'test', 'test_user'),
- (2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
- (0, 'quantities', 'dos', 'test_user'),
+ (2, 'dft.quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
+ (0, 'dft.quantities', 'dos', 'test_user'),
(2, 'external_id', 'external_2,external_3', 'other_test_user'),
(1, 'external_id', 'external_2', 'test_user'),
(1, 'external_id', 'external_2,external_3', 'test_user'),
@@ -879,17 +879,17 @@ class TestRepo():
assert statistics is not None
if quantity == 'system' and calcs != 0:
# for simplicity we only assert on quantities for this case
- assert 'system' in statistics
- assert len(statistics['system']) == 1
- assert value in statistics['system']
+ assert 'dft.system' in statistics
+ assert len(statistics['dft.system']) == 1
+ assert value in statistics['dft.system']
def test_search_exclude(self, api, example_elastic_calcs, no_warn):
- rv = api.get('/repo/?exclude=atoms,only_atoms')
+ rv = api.get('/repo/?exclude=dft.atoms,dft.only_atoms')
assert rv.status_code == 200
result = json.loads(rv.data)['results'][0]
- assert 'atoms' not in result
- assert 'only_atoms' not in result
- assert 'basis_set' in result
+ assert 'dft.atoms' not in result
+ assert 'dft.only_atoms' not in result
+ assert 'dft.basis_set' in result
metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]
@@ -950,8 +950,8 @@ class TestRepo():
assert len(results) == n_results
@pytest.mark.parametrize('first, order_by, order', [
- ('1', 'formula', -1), ('2', 'formula', 1),
- ('2', 'basis_set', -1), ('1', 'basis_set', 1),
+ ('1', 'dft.formula', -1), ('2', 'dft.formula', 1),
+ ('2', 'dft.basis_set', -1), ('1', 'dft.basis_set', 1),
(None, 'authors', -1)])
def test_search_order(self, api, example_elastic_calcs, no_warn, first, order_by, order):
rv = api.get('/repo/?order_by=%s&order=%d' % (order_by, order))
@@ -994,13 +994,13 @@ class TestRepo():
assert rv.status_code == 401
@pytest.mark.parametrize('calcs, quantity, value', [
- (2, 'system', 'bulk'),
- (0, 'system', 'atom'),
- (1, 'atoms', 'Br'),
- (1, 'atoms', 'Fe'),
+ (2, 'dft.system', 'bulk'),
+ (0, 'dft.system', 'atom'),
+ (1, 'dft.atoms', 'Br'),
+ (1, 'dft.atoms', 'Fe'),
(1, 'authors', 'Leonard Hofstadter'),
(2, 'files', 'test/mainfile.txt'),
- (0, 'quantities', 'dos')
+ (0, 'dft.quantities', 'dos')
])
def test_quantity_search(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
rv = api.get('/repo/quantity/%s' % quantity, headers=test_user_auth)
@@ -1014,7 +1014,7 @@ class TestRepo():
assert 0 == calcs
def test_quantity_search_after(self, api, example_elastic_calcs, no_warn, test_user_auth):
- rv = api.get('/repo/quantity/atoms?size=1')
+ rv = api.get('/repo/quantity/dft.atoms?size=1')
assert rv.status_code == 200
data = json.loads(rv.data)
@@ -1025,7 +1025,7 @@ class TestRepo():
value = list(quantity['values'].keys())[0]
while True:
- rv = api.get('/repo/quantity/atoms?size=1&after=%s' % after)
+ rv = api.get('/repo/quantity/dft.atoms?size=1&after=%s' % after)
assert rv.status_code == 200
data = json.loads(rv.data)
@@ -1040,7 +1040,10 @@ class TestRepo():
after = quantity['after']
def test_quantities_search(self, api, example_elastic_calcs, no_warn, test_user_auth):
- rv = api.get('/repo/quantities?%s' % urlencode(dict(quantities=['system', 'atoms'], size=1), doseq=True), headers=test_user_auth)
+ rv = api.get(
+ '/repo/quantities?%s' % urlencode(
+ dict(quantities=['dft.system', 'dft.atoms'], size=1), doseq=True),
+ headers=test_user_auth)
assert rv.status_code == 200
# TODO actual assertions
diff --git a/tests/app/test_app.py b/tests/app/test_app.py
index 2ede6169d152da80450fc270483d15354b6fc013..43a908b25276e3e3af8fe3c178fabf61e187d043 100644
--- a/tests/app/test_app.py
+++ b/tests/app/test_app.py
@@ -52,7 +52,7 @@ def test_internal_server_error_get(client, caplog):
rv = client.get('/api/test/ise?test_arg=value')
assert rv.status_code == 500
record = assert_log(caplog, 'error', 'internal server error')
- data = json.loads(record.message)
+ data = json.loads(record.msg)
assert data['blueprint'] == 'api'
assert data['endpoint'] == 'api.test_internal_server_error_resource'
@@ -67,7 +67,7 @@ def test_internal_server_error_post(client, caplog):
data=json.dumps(dict(test_arg='value')))
assert rv.status_code == 500
record = assert_log(caplog, 'error', 'internal server error')
- data = json.loads(record.message)
+ data = json.loads(record.msg)
assert data['blueprint'] == 'api'
assert data['endpoint'] == 'api.test_internal_server_error_resource'
diff --git a/tests/app/test_optimade.py b/tests/app/test_optimade.py
index 58327d69d5f2a55cb0c59847fda410e2578b39cf..ee2ce4ac56928304fc0fa7dd2d91359bb03c3157 100644
--- a/tests/app/test_optimade.py
+++ b/tests/app/test_optimade.py
@@ -36,7 +36,7 @@ def test_get_entry(published: Upload):
data = json.load(f)
assert 'OptimadeEntry' in data
search_result = search.SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
- assert 'optimade' in search_result
+ assert 'dft.optimade' in search_result
def test_no_optimade(meta_info, elastic, api):
diff --git a/tests/conftest.py b/tests/conftest.py
index 9ded2c2b68404be09701ebcc286dffa0dfbb218b..44e976602838caabc225c10f427697d9f5e211b6 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -49,7 +49,7 @@ example_files = [empty_file, example_file]
utils.ConsoleFormatter.short_format = True
-logging.Formatter = utils.ConsoleFormatter
+setattr(logging, 'Formatter', utils.ConsoleFormatter)
@pytest.fixture(scope="session")
diff --git a/tests/processing/test_data.py b/tests/processing/test_data.py
index 4ace9feecd4340bcd253ad1b0205870bd2e9ea6f..007e1d6d8b2417227a6b30fe3d2c15681e1ea9bf 100644
--- a/tests/processing/test_data.py
+++ b/tests/processing/test_data.py
@@ -383,8 +383,6 @@ def test_malicious_parser_task_failure(proc_infra, failure, test_user):
def test_ems_data(proc_infra, test_user):
-
-
upload = run_processing(('test_ems_upload', 'tests/data/proc/example_ems.zip'), test_user)
additional_keys = [
diff --git a/tests/test_cli.py b/tests/test_cli.py
index f5b530314421eee957e714acdba0fad98f1aba4d..8ce39dc0c0dace10acdcbabe4823f93ee323cc63 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -279,7 +279,7 @@ class TestClient:
new_search_results = new_search['results'][0]
for key in new_search_results.keys():
- if key not in ['upload_time', 'last_processing', 'labels']:
+ if key not in ['upload_time', 'last_processing', 'dft.labels']:
# There is a sub second change due to date conversions (?).
# Labels have arbitrary order.
assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])