Added more detailed test for optimade API.

nomad/app/optimade/endpoints.py

@@ -21,7 +21,8 @@ from nomad.metainfo.optimade import OptimadeEntry
 
 from .api import api, url
 from .models import json_api_single_response_model, entry_listing_endpoint_parser, Meta, \
-    Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser
+    Links, CalculationDataObject, single_entry_endpoint_parser, base_endpoint_parser,\
+    json_api_info_response_model
 from .filterparser import parse_filter, FilterException
 
 
@@ -33,7 +34,7 @@ def base_request_args():
     if request.args.get('response_format', 'json') != 'json':
         abort(400, 'Response format is not supported.')
 
-    properties_str = request.args.get('response_fields', None)
+    properties_str = request.args.get('request_fields', None)
     if properties_str is not None:
         return properties_str.split(',')
     return None
@@ -131,26 +132,19 @@ class CalculationInfo(Resource):
     @api.doc('calculations_info')
     @api.response(400, 'Invalid requests, e.g. bad parameter.')
     @api.expect(base_endpoint_parser, validate=True)
-    @api.marshal_with(json_api_single_response_model, skip_none=True, code=200)
+    @api.marshal_with(json_api_info_response_model, skip_none=True, code=200)
     def get(self):
         """ Returns information relating to the API implementation- """
         base_request_args()
 
         result = {
-            'type': 'info',
-            'id': 'calculation',
-            'attributes': {
-                'description': 'A calculations entry.',
-                # TODO non optimade, nomad specific properties
-                'properties': {
-                    attr.name: dict(description=attr.description)
-                    for attr in OptimadeEntry.m_def.all_properties.values()
-                },
-                'formats': ['json'],
-                'output_fields_by_format': {
-                    'json': OptimadeEntry.m_def.all_properties.keys()
-                }
-            }
+            'description': 'a calculation entry',
+            'properties': {
+                attr.name: dict(description=attr.description)
+                for attr in OptimadeEntry.m_def.all_properties.values()},
+            'formats': ['json'],
+            'output_fields_by_format': {
+                'json': OptimadeEntry.m_def.all_properties.keys()}
         }
 
         return dict(

nomad/app/optimade/models.py

@@ -212,15 +212,32 @@ json_api_data_object_model = api.model('DataObject', {
     # further optional fields: links, meta, relationships
 })
 
+json_api_calculation_info_model = api.model('CalculationInfo', {
+    'description': fields.String(
+        description='Description of the entry'),
+
+    'properties': fields.Raw(
+        description=('A dictionary describing queryable properties for this '
+                     'entry type, where each key is a property name')),
+
+    'formats': fields.List(
+        fields.String(),
+        required=True,
+        description='List of output formats available for this type of entry'),
+
+    'output_fields_by_format': fields.Raw(
+        description=('Dictionary of available output fields for this entry'
+                     'type, where the keys are the values of the formats list'
+                     'and the values are the keys of the properties dictionary'))
+
+})
+
 
 class CalculationDataObject:
     def __init__(self, calc: CalcWithMetadata, request_fields: Set[str] = None):
 
         def include(key):
-            if request_fields is None or \
-                    (key == 'optimade' and key in request_fields) or \
-                    (key != 'optimade' and '_nomad_%s' % key in request_fields):
-
+            if request_fields is None or (key in request_fields):
                 return True
 
             return False
@@ -266,6 +283,14 @@ json_api_list_response_model = api.inherit(
             description=('The list of returned response objects.'))
     })
 
+json_api_info_response_model = api.inherit(
+    'SingleResponse', json_api_response_model, {
+        'data': fields.Nested(
+            model=json_api_calculation_info_model,
+            required=True,
+            description=('The returned response object.'))
+    })
+
 
 base_endpoint_parser = api.parser()
 base_endpoint_parser.add_argument(

nomad/normalizing/optimade.py

@@ -16,6 +16,7 @@ from typing import Any, Dict
 import numpy as np
 import re
 import ase.data
+from string import ascii_uppercase
 
 from nomad.normalizing.normalizer import SystemBasedNormalizer
 from nomad.metainfo import units
@@ -77,9 +78,12 @@ class OptimadeNormalizer(SystemBasedNormalizer):
         optimade.chemical_formula_reduced = get_value('chemical_composition_reduced')
         optimade.chemical_formula_hill = get_value('chemical_composition_bulk_reduced')
         optimade.chemical_formula_descriptive = optimade.chemical_formula_hill
-        optimade.chemical_formula_anonymous = ''.join([
-            '%s' % element + (str(atom_counts[element]) if atom_counts[element] > 1 else '')
-            for element in optimade.elements])
+        optimade.chemical_formula_anonymous = ''
+        for i in range(len(optimade.elements)):
+            part = '%s' % ascii_uppercase[i % len(ascii_uppercase)]
+            if atom_counts[optimade.elements[i]] > 1:
+                part += str(atom_counts[optimade.elements[i]])
+            optimade.chemical_formula_anonymous += part
 
         # sites
         optimade.nsites = len(nomad_species)
@@ -94,7 +98,7 @@ class OptimadeNormalizer(SystemBasedNormalizer):
         for species_label in set(nomad_species):
             match = re.match(species_re, species_label)
 
-            element_label, index = match.groups(1), match.groups(2)
+            element_label = match.group(1)
 
             species = optimade.m_create(Species)
             species.name = species_label

tests/app/test_optimade.py

@@ -27,6 +27,7 @@ from nomad.app.optimade import parse_filter, url
 from tests.app.test_app import BlueprintClient
 from tests.test_normalizing import run_normalize
 from tests.conftest import clear_elastic
+from tests.utils import assert_exception
 
 
 @pytest.fixture(scope='session')
@@ -36,10 +37,8 @@ def api(nomad_app):
 
 def test_get_entry(published: Upload):
     calc_id = list(published.calcs)[0].calc_id
-
     with published.upload_files.archive_file(calc_id) as f:
         data = json.load(f)
-
     assert 'OptimadeEntry' in data
     search_result = search.SearchRequest().search_parameter('calc_id', calc_id).execute_paginated()['results'][0]
     assert 'optimade' in search_result
@@ -80,7 +79,8 @@ def create_test_structure(
         calc.optimade = None  # type: ignore
 
     proc.Calc.from_calc_with_metadata(calc).save()
-    search.Entry.from_calc_with_metadata(calc).save()
+    search_entry = search.Entry.from_calc_with_metadata(calc)
+    search_entry.save()
 
     assert proc.Calc.objects(calc_id__in=[calc_id]).count() == 1
 
@@ -128,6 +128,11 @@ def example_structures(meta_info, elastic_infra, mongo_infra):
     ('elements HAS ANY "C"', 1),
     ('elements HAS ONLY "C"', 0),
     ('elements HAS ONLY "H", "O"', 3),
+    ('nelements >= 2 AND elements HAS ONLY "H", "O"', 3),
+    ('nelements >= 2 AND elements HAS ALL "H", "O", "C"', 1),
+    ('nelements >= 2 AND NOT elements HAS ALL "H", "O", "C"', 3),
+    ('NOT nelements = 2 AND elements HAS ANY "H", "O", "C"', 1),
+    ('NOT nelements = 3 AND NOT elements HAS ONLY "H", "O"', 0),
     ('elements:elements_ratios HAS "H":>0.66', 2),
     ('elements:elements_ratios HAS ALL "O":>0.33', 3),
     ('elements:elements_ratios HAS ALL "O":>0.33,"O":<0.34', 2),
@@ -141,6 +146,13 @@ def example_structures(meta_info, elastic_infra, mongo_infra):
     ('chemical_formula_reduced STARTS WITH "H2"', 3),
     ('chemical_formula_reduced ENDS WITH "C"', 1),
     ('chemical_formula_reduced ENDS "C"', 1),
+    ('chemical_formula_hill CONTAINS "1"', 0),
+    ('chemical_formula_hill STARTS WITH "H" AND chemical_formula_hill ENDS WITH "O"', 3),
+    ('NOT chemical_formula_descriptive ENDS WITH "1"', 4),
+    ('chemical_formula_descriptive CONTAINS "C" AND NOT chemical_formula_descriptive STARTS WITH "O"', 1),
+    ('NOT chemical_formula_anonymous STARTS WITH "A"', 0),
+    ('chemical_formula_anonymous CONTAINS "AB2" AND chemical_formula_anonymous ENDS WITH "C"', 1),
+    ('nsites >=3 AND LENGTH elements = 2', 2),
     ('LENGTH elements = 2', 3),
     ('LENGTH elements = 3', 1),
     ('LENGTH dimension_types = 0', 3),
@@ -150,12 +162,19 @@ def example_structures(meta_info, elastic_infra, mongo_infra):
     ('nelements = 3 OR LENGTH dimension_types = 1', 2),
     ('nelements > 1 OR LENGTH dimension_types = 1 AND nelements = 2', 4),
     ('(nelements > 1 OR LENGTH dimension_types = 1) AND nelements = 2', 3),
-    ('NOT LENGTH dimension_types = 1', 3)
+    ('NOT LENGTH dimension_types = 1', 3),
+    ('LENGTH nelements = 1', -1),
+    ('chemical_formula_anonymous starts with "A"', -1),
+    ('elements HAS ONY "H", "O"', -1)
 ])
 def test_optimade_parser(example_structures, query, results):
-    query = parse_filter(query)
-    result = search.SearchRequest(query=query).execute_paginated()
-    assert result['pagination']['total'] == results
+    if results >= 0:
+        query = parse_filter(query)
+        result = search.SearchRequest(query=query).execute_paginated()
+        assert result['pagination']['total'] == results
+    else:
+        with assert_exception():
+            query = parse_filter(query)
 
 
 def test_url():
@@ -166,42 +185,72 @@ def test_list_endpoint(api, example_structures):
     rv = api.get('/calculations')
     assert rv.status_code == 200
     data = json.loads(rv.data)
-    # TODO replace with real assertions
-    # print(json.dumps(data, indent=2))
+    for entry in ['data', 'links', 'meta']:
+        assert entry in data
+    assert len(data['data']) == 4
+
+
+def assert_eq_attrib(data, key, ref, item=None):
+    if item is None:
+        assert data['data']['attributes'][key] == ref
+    else:
+        assert data['data'][item]['attributes'][key] == ref
 
 
 def test_list_endpoint_request_fields(api, example_structures):
     rv = api.get('/calculations?request_fields=nelements,elements')
     assert rv.status_code == 200
     data = json.loads(rv.data)
-    # TODO replace with real assertions
-    # print(json.dumps(data, indent=2))
+    ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']]
+    for i in range(len(data['data'])):
+        rf = list(data['data'][i]['attributes'].keys())
+        rf.sort()
+        assert rf == ['elements', 'nelements']
+        assert_eq_attrib(data, 'elements', ref_elements[i], i)
+        assert_eq_attrib(data, 'nelements', len(ref_elements[i]), i)
+
+
+def test_single_endpoint_request_fields(api, example_structures):
+    rv = api.get('/calculations/%s?request_fields=nelements,elements' % 'test_calc_id_1')
+    assert rv.status_code == 200
+    data = json.loads(rv.data)
+    ref_elements = ['H', 'O']
+    rf = list(data['data']['attributes'].keys())
+    assert rf == ['elements', 'nelements']
+    assert_eq_attrib(data, 'elements', ref_elements)
+    assert_eq_attrib(data, 'nelements', len(ref_elements))
 
 
 def test_single_endpoint(api, example_structures):
     rv = api.get('/calculations/%s' % 'test_calc_id_1')
     assert rv.status_code == 200
     data = json.loads(rv.data)
-    # TODO replace with real assertions
-    # print(json.dumps(data, indent=2))
+    for key in ['type', 'id', 'attributes']:
+        assert key in data['data']
+    fields = ['elements', 'nelements', 'elements_ratios',
+              'chemical_formula_descriptive', 'chemical_formula_reduced',
+              'chemical_formula_hill', 'chemical_formula_anonymous',
+              'dimension_types', 'lattice_vectors', 'cartesian_site_positions',
+              'nsites', 'species_at_sites', 'species']
+    for field in fields:
+        assert field in data['data']['attributes']
 
 
 def test_base_info_endpoint(api):
     rv = api.get('/info')
     assert rv.status_code == 200
     data = json.loads(rv.data)
-    # TODO replace with real assertions
-    # print(json.dumps(data, indent=2))
+    for key in ['type', 'id', 'attributes']:
+        assert key in data['data']
+    assert data['data']['type'] == 'info'
+    assert data['data']['id'] == '/'
 
 
 def test_calculation_info_endpoint(api):
     rv = api.get('/info/calculation')
     assert rv.status_code == 200
     data = json.loads(rv.data)
-    # TODO replace with real assertions
-    # print(json.dumps(data, indent=2))
-
+    for key in ['description', 'properties', 'formats', 'output_fields_by_format']:
+        assert key in data['data']
 
-# TODO test single with request_fields
-# TODO test errors
 # TODO test response format (deny everything but json)