From d54c924deba5c0c7be125a33fb272e5164c4cfa2 Mon Sep 17 00:00:00 2001
From: Alvin Noe Ladines <ladinesalvinnoe@gmail.com>
Date: Fri, 11 Oct 2019 14:46:32 +0200
Subject: [PATCH] Added more detailed test for optimade API.
---
nomad/app/optimade/endpoints.py | 28 ++++-------
nomad/app/optimade/models.py | 33 ++++++++++--
nomad/normalizing/optimade.py | 12 +++--
tests/app/test_optimade.py | 89 +++++++++++++++++++++++++--------
4 files changed, 117 insertions(+), 45 deletions(-)
diff --git a/nomad/app/optimade/endpoints.py b/nomad/app/optimade/endpoints.py
index 375e8e5660..c2d22b2dd2 100644
--- a/nomad/app/optimade/endpoints.py
+++ b/nomad/app/optimade/endpoints.py
@@ -21,7 +21,8 @@ from nomad.metainfo.optimade import OptimadeEntry
from .api import api, url
from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
- Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser
+ Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser,\
+ json_api_info_response_model
from .filterparser import parse_filter, FilterException
@@ -33,7 +34,7 @@ def base_request_args():
if request.args.get('response_format', 'json') != 'json':
abort(400, 'Response format is not supported.')
- properties_str = request.args.get('response_fields', None)
+ properties_str = request.args.get('request_fields', None)
if properties_str is not None:
return properties_str.split(',')
return None
@@ -131,26 +132,19 @@ class CalculationInfo(Resource):
@api.doc('calculations_info')
@api.response(400, 'Invalid requests, e.g. bad parameter.')
@api.expect(base_endpoint_parser, validate=True)
- @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
+ @api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
def get(self):
""" Returns information relating to the API implementation- """
base_request_args()
result = {
- 'type': 'info',
- 'id': 'calculation',
- 'attributes': {
- 'description': 'A calculations entry.',
- # TODO non optimade, nomad specific properties
- 'properties': {
- attr.name: dict(description=attr.description)
- for attr in OptimadeEntry.m_def.all_properties.values()
- },
- 'formats': ['json'],
- 'output_fields_by_format': {
- 'json': OptimadeEntry.m_def.all_properties.keys()
- }
- }
+ 'description': 'a calculation entry',
+ 'properties': {
+ attr.name: dict(description=attr.description)
+ for attr in OptimadeEntry.m_def.all_properties.values()},
+ 'formats': ['json'],
+ 'output_fields_by_format': {
+ 'json': OptimadeEntry.m_def.all_properties.keys()}
}
return dict(
diff --git a/nomad/app/optimade/models.py b/nomad/app/optimade/models.py
index 32d47f56ca..3fbef7e238 100644
--- a/nomad/app/optimade/models.py
+++ b/nomad/app/optimade/models.py
@@ -212,15 +212,32 @@ json_api_data_object_model = api.model('DataObject', {
# further optional fields: links, meta, relationships
})
+json_api_calculation_info_model = api.model('CalculationInfo', {
+ 'description': fields.String(
+ description='Description of the entry'),
+
+ 'properties': fields.Raw(
+ description=('A dictionary describing queryable properties for this '
+ 'entry type, where each key is a property name')),
+
+ 'formats': fields.List(
+ fields.String(),
+ required=True,
+ description='List of output formats available for this type of entry'),
+
+ 'output_fields_by_format': fields.Raw(
+ description=('Dictionary of available output fields for this entry'
+ 'type, where the keys are the values of the formats list'
+ 'and the values are the keys of the properties dictionary'))
+
+})
+
class CalculationDataObject:
def __init__(self, calc: CalcWithMetadata, request_fields: Set[str] = None):
def include(key):
- if request_fields is None or \
- (key == 'optimade' and key in request_fields) or \
- (key != 'optimade' and '_nomad_%s' % key in request_fields):
-
+ if request_fields is None or (key in request_fields):
return True
return False
@@ -266,6 +283,14 @@ json_api_list_response_model = api.inherit(
description=('The list of returned response objects.'))
})
+json_api_info_response_model = api.inherit(
+ 'SingleResponse', json_api_response_model, {
+ 'data': fields.Nested(
+ model=json_api_calculation_info_model,
+ required=True,
+ description=('The returned response object.'))
+ })
+
base_endpoint_parser = api.parser()
base_endpoint_parser.add_argument(
diff --git a/nomad/normalizing/optimade.py b/nomad/normalizing/optimade.py
index e439c9a2ee..c91754a75d 100644
--- a/nomad/normalizing/optimade.py
+++ b/nomad/normalizing/optimade.py
@@ -16,6 +16,7 @@ from typing import Any, Dict
import numpy as np
import re
import ase.data
+from string import ascii_uppercase
from nomad.normalizing.normalizer import SystemBasedNormalizer
from nomad.metainfo import units
@@ -77,9 +78,12 @@ class OptimadeNormalizer(SystemBasedNormalizer):
optimade.chemical_formula_reduced = get_value('chemical_composition_reduced')
optimade.chemical_formula_hill = get_value('chemical_composition_bulk_reduced')
optimade.chemical_formula_descriptive = optimade.chemical_formula_hill
- optimade.chemical_formula_anonymous = ''.join([
- '%s' % element + (str(atom_counts[element]) if atom_counts[element] > 1 else '')
- for element in optimade.elements])
+ optimade.chemical_formula_anonymous = ''
+ for i in range(len(optimade.elements)):
+ part = '%s' % ascii_uppercase[i % len(ascii_uppercase)]
+ if atom_counts[optimade.elements[i]] > 1:
+ part += str(atom_counts[optimade.elements[i]])
+ optimade.chemical_formula_anonymous += part
# sites
optimade.nsites = len(nomad_species)
@@ -94,7 +98,7 @@ class OptimadeNormalizer(SystemBasedNormalizer):
for species_label in set(nomad_species):
match = re.match(species_re, species_label)
- element_label, index = match.groups(1), match.groups(2)
+ element_label = match.group(1)
species = optimade.m_create(Species)
species.name = species_label
diff --git a/tests/app/test_optimade.py b/tests/app/test_optimade.py
index 9b73107e28..d862d1ada8 100644
--- a/tests/app/test_optimade.py
+++ b/tests/app/test_optimade.py
@@ -27,6 +27,7 @@ from nomad.app.optimade import parse_filter, url
from tests.app.test_app import BlueprintClient
from tests.test_normalizing import run_normalize
from tests.conftest import clear_elastic
+from tests.utils import assert_exception
@pytest.fixture(scope='session')
@@ -36,10 +37,8 @@ def api(nomad_app):
def test_get_entry(published: Upload):
calc_id = list(published.calcs)[0].calc_id
-
with published.upload_files.archive_file(calc_id) as f:
data = json.load(f)
-
assert 'OptimadeEntry' in data
search_result = search.SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
assert 'optimade' in search_result
@@ -80,7 +79,8 @@ def create_test_structure(
calc.optimade = None # type: ignore
proc.Calc.from_calc_with_metadata(calc).save()
- search.Entry.from_calc_with_metadata(calc).save()
+ search_entry = search.Entry.from_calc_with_metadata(calc)
+ search_entry.save()
assert proc.Calc.objects(calc_id__in=[calc_id]).count() == 1
@@ -128,6 +128,11 @@ def example_structures(meta_info, elastic_infra, mongo_infra):
('elements HAS ANY "C"', 1),
('elements HAS ONLY "C"', 0),
('elements HAS ONLY "H", "O"', 3),
+ ('nelements >= 2 AND elements HAS ONLY "H", "O"', 3),
+ ('nelements >= 2 AND elements HAS ALL "H", "O", "C"', 1),
+ ('nelements >= 2 AND NOT elements HAS ALL "H", "O", "C"', 3),
+ ('NOT nelements = 2 AND elements HAS ANY "H", "O", "C"', 1),
+ ('NOT nelements = 3 AND NOT elements HAS ONLY "H", "O"', 0),
('elements:elements_ratios HAS "H":>0.66', 2),
('elements:elements_ratios HAS ALL "O":>0.33', 3),
('elements:elements_ratios HAS ALL "O":>0.33,"O":<0.34', 2),
@@ -141,6 +146,13 @@ def example_structures(meta_info, elastic_infra, mongo_infra):
('chemical_formula_reduced STARTS WITH "H2"', 3),
('chemical_formula_reduced ENDS WITH "C"', 1),
('chemical_formula_reduced ENDS "C"', 1),
+ ('chemical_formula_hill CONTAINS "1"', 0),
+ ('chemical_formula_hill STARTS WITH "H" AND chemical_formula_hill ENDS WITH "O"', 3),
+ ('NOT chemical_formula_descriptive ENDS WITH "1"', 4),
+ ('chemical_formula_descriptive CONTAINS "C" AND NOT chemical_formula_descriptive STARTS WITH "O"', 1),
+ ('NOT chemical_formula_anonymous STARTS WITH "A"', 0),
+ ('chemical_formula_anonymous CONTAINS "AB2" AND chemical_formula_anonymous ENDS WITH "C"', 1),
+ ('nsites >=3 AND LENGTH elements = 2', 2),
('LENGTH elements = 2', 3),
('LENGTH elements = 3', 1),
('LENGTH dimension_types = 0', 3),
@@ -150,12 +162,19 @@ def example_structures(meta_info, elastic_infra, mongo_infra):
('nelements = 3 OR LENGTH dimension_types = 1', 2),
('nelements > 1 OR LENGTH dimension_types = 1 AND nelements = 2', 4),
('(nelements > 1 OR LENGTH dimension_types = 1) AND nelements = 2', 3),
- ('NOT LENGTH dimension_types = 1', 3)
+ ('NOT LENGTH dimension_types = 1', 3),
+ ('LENGTH nelements = 1', -1),
+ ('chemical_formula_anonymous starts with "A"', -1),
+ ('elements HAS ONY "H", "O"', -1)
])
def test_optimade_parser(example_structures, query, results):
- query = parse_filter(query)
- result = search.SearchRequest(query=query).execute_paginated()
- assert result['pagination']['total'] == results
+ if results >= 0:
+ query = parse_filter(query)
+ result = search.SearchRequest(query=query).execute_paginated()
+ assert result['pagination']['total'] == results
+ else:
+ with assert_exception():
+ query = parse_filter(query)
def test_url():
@@ -166,42 +185,72 @@ def test_list_endpoint(api, example_structures):
rv = api.get('/calculations')
assert rv.status_code == 200
data = json.loads(rv.data)
- # TODO replace with real assertions
- # print(json.dumps(data, indent=2))
+ for entry in ['data', 'links', 'meta']:
+ assert entry in data
+ assert len(data['data']) == 4
+
+
+def assert_eq_attrib(data, key, ref, item=None):
+ if item is None:
+ assert data['data']['attributes'][key] == ref
+ else:
+ assert data['data'][item]['attributes'][key] == ref
def test_list_endpoint_request_fields(api, example_structures):
rv = api.get('/calculations?request_fields=nelements,elements')
assert rv.status_code == 200
data = json.loads(rv.data)
- # TODO replace with real assertions
- # print(json.dumps(data, indent=2))
+ ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']]
+ for i in range(len(data['data'])):
+ rf = list(data['data'][i]['attributes'].keys())
+ rf.sort()
+ assert rf == ['elements', 'nelements']
+ assert_eq_attrib(data, 'elements', ref_elements[i], i)
+ assert_eq_attrib(data, 'nelements', len(ref_elements[i]), i)
+
+
+def test_single_endpoint_request_fields(api, example_structures):
+ rv = api.get('/calculations/%s?request_fields=nelements,elements' % 'test_calc_id_1')
+ assert rv.status_code == 200
+ data = json.loads(rv.data)
+ ref_elements = ['H', 'O']
+ rf = list(data['data']['attributes'].keys())
+ assert rf == ['elements', 'nelements']
+ assert_eq_attrib(data, 'elements', ref_elements)
+ assert_eq_attrib(data, 'nelements', len(ref_elements))
def test_single_endpoint(api, example_structures):
rv = api.get('/calculations/%s' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
- # TODO replace with real assertions
- # print(json.dumps(data, indent=2))
+ for key in ['type', 'id', 'attributes']:
+ assert key in data['data']
+ fields = ['elements', 'nelements', 'elements_ratios',
+ 'chemical_formula_descriptive', 'chemical_formula_reduced',
+ 'chemical_formula_hill', 'chemical_formula_anonymous',
+ 'dimension_types', 'lattice_vectors', 'cartesian_site_positions',
+ 'nsites', 'species_at_sites', 'species']
+ for field in fields:
+ assert field in data['data']['attributes']
def test_base_info_endpoint(api):
rv = api.get('/info')
assert rv.status_code == 200
data = json.loads(rv.data)
- # TODO replace with real assertions
- # print(json.dumps(data, indent=2))
+ for key in ['type', 'id', 'attributes']:
+ assert key in data['data']
+ assert data['data']['type'] == 'info'
+ assert data['data']['id'] == '/'
def test_calculation_info_endpoint(api):
rv = api.get('/info/calculation')
assert rv.status_code == 200
data = json.loads(rv.data)
- # TODO replace with real assertions
- # print(json.dumps(data, indent=2))
-
+ for key in ['description', 'properties', 'formats', 'output_fields_by_format']:
+ assert key in data['data']
-# TODO test single with request_fields
-# TODO test errors
# TODO test response format (deny everything but json)
--
GitLab