Commit d9391c8a authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

All tests working after domain refactor.

parent eac09baa
......@@ -69,25 +69,30 @@ class InfoResource(Resource):
@api.marshal_with(info_model, skip_none=True, code=200, description='Info send')
def get(self):
""" Return information about the nomad backend and its configuration. """
codes = [
parser.code_name
for parser in parsing.parser_dict.values()
if isinstance(parser, parsing.MatchingParser) and parser.domain == 'dft']
return {
'parsers': [
key[key.index('/') + 1:]
for key in parsing.parser_dict.keys()],
'codes': sorted(set([
parser.code_name
for parser in parsing.parser_dict.values()
if isinstance(parser, parsing.MatchingParser) and parser.domain == datamodel.Domain.instance.name]), key=lambda x: x.lower()),
'codes': sorted(set(codes), key=lambda x: x.lower()),
'normalizers': [normalizer.__name__ for normalizer in normalizing.normalizers],
'domain': {
'name': datamodel.Domain.instance.name,
'quantities': [quantity for quantity in datamodel.Domain.instance.quantities.values()],
'metrics_names': datamodel.Domain.instance.metrics_names,
'aggregations_names': datamodel.Domain.instance.aggregations_names,
'metainfo': {
'all_package': datamodel.Domain.instance.metainfo_all_package,
'root_sections': datamodel.Domain.instance.root_sections
'domains': [
{
'name': domain.name,
'quantities': [quantity for quantity in domain.quantities.values()],
'metrics_names': domain.metrics_names,
'aggregations_names': domain.aggregations_names,
'metainfo': {
'all_package': domain.metainfo_all_package,
'root_sections': domain.root_sections
}
}
},
for domain in datamodel.Domain.instances.values()
],
'version': config.version,
'release': config.release,
'git': {
......
......@@ -148,7 +148,8 @@ class RepoCalcsResource(Resource):
The search will return aggregations on a predefined set of quantities. Aggregations
will tell you what quantity values exist and how many entries match those values.
Ordering is determined by ``order_by`` and ``order`` parameters.
Ordering is determined by ``order_by`` and ``order`` parameters. Default is
``upload_time`` in decending order.
"""
try:
......@@ -162,7 +163,7 @@ class RepoCalcsResource(Resource):
page = args.get('page', 1)
per_page = args.get('per_page', 10 if not scroll else 1000)
order = args.get('order', -1)
order_by = args.get('order_by', 'formula')
order_by = args.get('order_by', 'upload_time')
date_histogram = args.get('date_histogram', False)
metrics: List[str] = request.args.getlist('metrics')
......@@ -594,9 +595,8 @@ class RepoQuantityResource(Resource):
except AssertionError:
abort(400, message='invalid size')
search_request.quantity(quantity, size=size, after=after)
try:
search_request.quantity(quantity, size=size, after=after)
results = search_request.execute()
quantities = results.pop('quantities')
results['quantity'] = quantities[quantity]
......
......@@ -43,7 +43,7 @@ def base_request_args():
def base_search_request():
""" Creates a search request for all public and optimade enabled data. """
return search.SearchRequest().owner('all', None).query(
Q('exists', field='optimade.nelements')) # TODO use the elastic annotations when done
Q('exists', field='dft.optimade.nelements')) # TODO use the elastic annotations when done
@ns.route('/calculations')
......
......@@ -26,7 +26,7 @@ class FilterException(Exception):
quantities: Dict[str, Quantity] = {
q.name: Quantity(
q.name, es_field='optimade.%s' % q.name,
q.name, es_field='dft.optimade.%s' % q.name,
elastic_mapping_type=q.m_annotations['elastic']['type'])
for q in OptimadeEntry.m_def.all_quantities.values()
......@@ -34,7 +34,7 @@ quantities: Dict[str, Quantity] = {
quantities['elements'].length_quantity = quantities['nelements']
quantities['dimension_types'].length_quantity = quantities['dimension_types']
quantities['elements'].has_only_quantity = Quantity(name='only_atoms')
quantities['elements'].has_only_quantity = Quantity(name='dft.only_atoms')
quantities['elements'].nested_quantity = quantities['elements_ratios']
quantities['elements_ratios'].nested_quantity = quantities['elements_ratios']
......
......@@ -50,7 +50,7 @@ def uploads(ctx, user: str, staging: bool, processing: bool, outdated: bool, cod
query &= Q(upload_id__in=uploads)
if code is not None and len(code) > 0:
code_queries = [es.Q('match', code_name=code_name) for code_name in code]
code_queries = [es.Q('match', **{'dft.code_name': code_name}) for code_name in code]
code_query = es.Q('bool', should=code_queries, minimum_should_match=1)
code_search = es.Search(index=config.elastic.index_name)
......
......@@ -34,7 +34,7 @@ def codes(client, minimum=1, **kwargs):
data = client.repo.search(per_page=1, **kwargs).response().result
x_values = sorted([
code for code, values in data.statistics['code_name'].items()
code for code, values in data.statistics['dft.code_name'].items()
if code != 'not processed' and values.get('calculations', 1000) >= minimum], key=lambda x: x.lower())
return data.statistics, x_values, 'code_name', 'code'
......@@ -61,7 +61,7 @@ def error_fig(client):
return {
code: values[metric]
for code, values in result.quantities['code_name'].items()
for code, values in result.quantities['dft.code_name'].items()
if code != 'not processed' and (not labels or code in labels) > 0}
# get the data
......@@ -414,7 +414,7 @@ def statistics_table(html, geometries, public_path):
'total', 'all', 'code_runs')
phonons = get_statistic(
client.repo.search(per_page=1, code_name='Phonopy').response().result,
client.repo.search(per_page=1, **{'dft.code_name': 'Phonopy'}).response().result,
'total', 'all', 'code_runs')
# files and sized
......
......@@ -335,4 +335,4 @@ Domain(
groups=dict(
groups=('group_hash', 'groups')),
default_statistics=[
'atoms', 'basis_set', 'xc_functional', 'system', 'crystal_system', 'code_name'])
'dft.atoms', 'dft.basis_set', 'dft.xc_functional', 'dft.system', 'dft.crystal_system', 'dft.code_name'])
......@@ -815,7 +815,7 @@ class TestRepo():
data = self.assert_search(rv, calcs)
results = data.get('results', None)
if calcs > 0:
for key in ['uploader', 'calc_id', 'formula', 'upload_id']:
for key in ['uploader', 'calc_id', 'dft.formula', 'upload_id']:
assert key in results[0]
@pytest.mark.parametrize('calcs, start, end', [
......@@ -844,22 +844,22 @@ class TestRepo():
self.assert_search(rv, calcs)
@pytest.mark.parametrize('calcs, quantity, value, user', [
(2, 'system', 'bulk', 'test_user'),
(0, 'system', 'atom', 'test_user'),
(1, 'atoms', 'Br', 'test_user'),
(1, 'atoms', 'Fe', 'test_user'),
(0, 'atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
(0, 'only_atoms', ['Br', 'Si'], 'test_user'),
(1, 'only_atoms', ['Fe'], 'test_user'),
(1, 'only_atoms', ['Br', 'K', 'Si'], 'test_user'),
(1, 'only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(2, 'dft.system', 'bulk', 'test_user'),
(0, 'dft.system', 'atom', 'test_user'),
(1, 'dft.atoms', 'Br', 'test_user'),
(1, 'dft.atoms', 'Fe', 'test_user'),
(0, 'dft.atoms', ['Fe', 'Br', 'A', 'B'], 'test_user'),
(0, 'dft.only_atoms', ['Br', 'Si'], 'test_user'),
(1, 'dft.only_atoms', ['Fe'], 'test_user'),
(1, 'dft.only_atoms', ['Br', 'K', 'Si'], 'test_user'),
(1, 'dft.only_atoms', ['Br', 'Si', 'K'], 'test_user'),
(1, 'comment', 'specific', 'test_user'),
(1, 'authors', 'Leonard Hofstadter', 'test_user'),
(2, 'files', 'test/mainfile.txt', 'test_user'),
(2, 'paths', 'mainfile.txt', 'test_user'),
(2, 'paths', 'test', 'test_user'),
(2, 'quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
(0, 'quantities', 'dos', 'test_user'),
(2, 'dft.quantities', ['wyckoff_letters_primitive', 'hall_number'], 'test_user'),
(0, 'dft.quantities', 'dos', 'test_user'),
(2, 'external_id', 'external_2,external_3', 'other_test_user'),
(1, 'external_id', 'external_2', 'test_user'),
(1, 'external_id', 'external_2,external_3', 'test_user'),
......@@ -879,17 +879,17 @@ class TestRepo():
assert statistics is not None
if quantity == 'system' and calcs != 0:
# for simplicity we only assert on quantities for this case
assert 'system' in statistics
assert len(statistics['system']) == 1
assert value in statistics['system']
assert 'dft.system' in statistics
assert len(statistics['dft.system']) == 1
assert value in statistics['dft.system']
def test_search_exclude(self, api, example_elastic_calcs, no_warn):
rv = api.get('/repo/?exclude=atoms,only_atoms')
rv = api.get('/repo/?exclude=dft.atoms,dft.only_atoms')
assert rv.status_code == 200
result = json.loads(rv.data)['results'][0]
assert 'atoms' not in result
assert 'only_atoms' not in result
assert 'basis_set' in result
assert 'dft.atoms' not in result
assert 'dft.only_atoms' not in result
assert 'dft.basis_set' in result
metrics_permutations = [[], search.metrics_names] + [[metric] for metric in search.metrics_names]
......@@ -950,8 +950,8 @@ class TestRepo():
assert len(results) == n_results
@pytest.mark.parametrize('first, order_by, order', [
('1', 'formula', -1), ('2', 'formula', 1),
('2', 'basis_set', -1), ('1', 'basis_set', 1),
('1', 'dft.formula', -1), ('2', 'dft.formula', 1),
('2', 'dft.basis_set', -1), ('1', 'dft.basis_set', 1),
(None, 'authors', -1)])
def test_search_order(self, api, example_elastic_calcs, no_warn, first, order_by, order):
rv = api.get('/repo/?order_by=%s&order=%d' % (order_by, order))
......@@ -994,13 +994,13 @@ class TestRepo():
assert rv.status_code == 401
@pytest.mark.parametrize('calcs, quantity, value', [
(2, 'system', 'bulk'),
(0, 'system', 'atom'),
(1, 'atoms', 'Br'),
(1, 'atoms', 'Fe'),
(2, 'dft.system', 'bulk'),
(0, 'dft.system', 'atom'),
(1, 'dft.atoms', 'Br'),
(1, 'dft.atoms', 'Fe'),
(1, 'authors', 'Leonard Hofstadter'),
(2, 'files', 'test/mainfile.txt'),
(0, 'quantities', 'dos')
(0, 'dft.quantities', 'dos')
])
def test_quantity_search(self, api, example_elastic_calcs, no_warn, test_user_auth, calcs, quantity, value):
rv = api.get('/repo/quantity/%s' % quantity, headers=test_user_auth)
......@@ -1014,7 +1014,7 @@ class TestRepo():
assert 0 == calcs
def test_quantity_search_after(self, api, example_elastic_calcs, no_warn, test_user_auth):
rv = api.get('/repo/quantity/atoms?size=1')
rv = api.get('/repo/quantity/dft.atoms?size=1')
assert rv.status_code == 200
data = json.loads(rv.data)
......@@ -1025,7 +1025,7 @@ class TestRepo():
value = list(quantity['values'].keys())[0]
while True:
rv = api.get('/repo/quantity/atoms?size=1&after=%s' % after)
rv = api.get('/repo/quantity/dft.atoms?size=1&after=%s' % after)
assert rv.status_code == 200
data = json.loads(rv.data)
......@@ -1040,7 +1040,10 @@ class TestRepo():
after = quantity['after']
def test_quantities_search(self, api, example_elastic_calcs, no_warn, test_user_auth):
rv = api.get('/repo/quantities?%s' % urlencode(dict(quantities=['system', 'atoms'], size=1), doseq=True), headers=test_user_auth)
rv = api.get(
'/repo/quantities?%s' % urlencode(
dict(quantities=['dft.system', 'dft.atoms'], size=1), doseq=True),
headers=test_user_auth)
assert rv.status_code == 200
# TODO actual assertions
......
......@@ -52,7 +52,7 @@ def test_internal_server_error_get(client, caplog):
rv = client.get('/api/test/ise?test_arg=value')
assert rv.status_code == 500
record = assert_log(caplog, 'error', 'internal server error')
data = json.loads(record.message)
data = json.loads(record.msg)
assert data['blueprint'] == 'api'
assert data['endpoint'] == 'api.test_internal_server_error_resource'
......@@ -67,7 +67,7 @@ def test_internal_server_error_post(client, caplog):
data=json.dumps(dict(test_arg='value')))
assert rv.status_code == 500
record = assert_log(caplog, 'error', 'internal server error')
data = json.loads(record.message)
data = json.loads(record.msg)
assert data['blueprint'] == 'api'
assert data['endpoint'] == 'api.test_internal_server_error_resource'
......
......@@ -36,7 +36,7 @@ def test_get_entry(published: Upload):
data = json.load(f)
assert 'OptimadeEntry' in data
search_result = search.SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
assert 'optimade' in search_result
assert 'dft.optimade' in search_result
def test_no_optimade(meta_info, elastic, api):
......
......@@ -49,7 +49,7 @@ example_files = [empty_file, example_file]
utils.ConsoleFormatter.short_format = True
logging.Formatter = utils.ConsoleFormatter
setattr(logging, 'Formatter', utils.ConsoleFormatter)
@pytest.fixture(scope="session")
......
......@@ -383,8 +383,6 @@ def test_malicious_parser_task_failure(proc_infra, failure, test_user):
def test_ems_data(proc_infra, test_user):
upload = run_processing(('test_ems_upload', 'tests/data/proc/example_ems.zip'), test_user)
additional_keys = [
......
......@@ -279,7 +279,7 @@ class TestClient:
new_search_results = new_search['results'][0]
for key in new_search_results.keys():
if key not in ['upload_time', 'last_processing', 'labels']:
if key not in ['upload_time', 'last_processing', 'dft.labels']:
# There is a sub second change due to date conversions (?).
# Labels have arbitrary order.
assert json.dumps(new_search_results[key]) == json.dumps(ref_search_results[key])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment