Merge branch 'v0.10.0' into 'master'

Merge v0.10.0 into master for release

Closes #475, #484, #497, #492, #498, and #500

See merge request !283

tests/app/test_api.py → tests/app/flask/test_api.py

@@ -30,22 +30,23 @@ import base64
 import itertools
 from hashlib import md5
 
-from nomad.app.common import rfc3339DateTime
-from nomad.app.api.auth import generate_upload_token
+from nomad.app.flask.common import rfc3339DateTime
+from nomad.app.flask.api.auth import generate_upload_token
 from nomad import search, files, config, utils, infrastructure
 from nomad.metainfo import search_extension
 from nomad.files import UploadFiles, PublicUploadFiles
 from nomad.processing import Upload, Calc, SUCCESS
 from nomad.datamodel import EntryMetadata, User, Dataset
 
-from tests.conftest import create_auth_headers, clear_elastic, clear_raw_files
 from tests.test_files import example_file, example_file_mainfile, example_file_contents
 from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
 from tests.test_search import assert_search_upload
 from tests.processing import test_data as test_processing
-from tests.processing.test_data import oasis_publishable_upload
+from tests.processing.test_data import oasis_publishable_upload  # pylint: disable=unused-import
+from tests.conftest import clear_elastic, clear_raw_files
 
-from tests.app.test_app import BlueprintClient
+from ..conftest import create_auth_headers
+from .test_app import BlueprintClient
 
 logger = utils.get_logger(__name__)
 
@@ -1033,6 +1034,12 @@ class TestRepo():
         assert data['code']['python'] is not None
         assert data['code']['clientlib'] is not None
 
+    def test_get_defaults(self, api, example_elastic_calcs, no_warn, test_user_auth):
+        rv = api.get('/repo/0/1', headers=test_user_auth)
+        assert rv.status_code == 200
+        data = rv.json
+        assert data['license'] is not None
+
     def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
         rv = api.get('/repo/0/1', headers=other_test_user_auth)
         assert rv.status_code == 200
@@ -1169,7 +1176,7 @@ class TestRepo():
             assert value in statistics['dft.system']
 
     def test_search_exclude(self, api, example_elastic_calcs, no_warn):
-        rv = api.get('/repo/?exclude=atoms,only_atoms')
+        rv = api.get('/repo/?exclude=atoms,only_atoms,dft.optimade,dft.quantities')
         assert rv.status_code == 200
         result = search.flat(json.loads(rv.data)['results'][0])
         assert 'atoms' not in result
@@ -1513,7 +1520,7 @@ class TestEditRepo():
 
     @pytest.fixture(autouse=True)
     def example_data(self, class_api, test_user, other_test_user, raw_files):
-        from tests.app.utils import Upload
+        from .utils import Upload
 
         uploads = {}
         for i in range(1, 4):

tests/app/test_api_encyclopedia.py → tests/app/flask/test_api_encyclopedia.py

@@ -25,7 +25,7 @@ from nomad import config
 from nomad.cli import cli
 from nomad import processing as proc, infrastructure
 
-from tests.app.test_app import BlueprintClient
+from tests.app.flask.test_app import BlueprintClient
 
 silicon_id = "fh3UBjhUVm4nxzeRd2JJuqw5oXYa"
 

tests/app/test_app.py → tests/app/flask/test_app.py

@@ -21,10 +21,8 @@
 import json
 import pytest
 
-from nomad import config
-
 from tests.utils import assert_log
-from tests.app import resource  # pylint: disable=unused-import
+from . import resource  # pylint: disable=unused-import
 
 
 class BlueprintClient():
@@ -82,7 +80,7 @@ def test_internal_server_error_post(client, caplog):
     assert data['json']['test_arg'] == 'value'
 
 
-@pytest.mark.parametrize('api', ['api', 'optimade'])
+@pytest.mark.parametrize('api', ['api'])
 def test_swagger(client, api):
     rv = client.get('/%s/swagger.json' % api)
     assert rv.status_code == 200

tests/app/test_dcat.py → tests/app/flask/test_dcat.py

@@ -21,10 +21,10 @@ from datetime import datetime
 
 from nomad import infrastructure, config
 from nomad.datamodel import EntryMetadata
-from nomad.app.dcat.mapping import Mapping
+from nomad.app.flask.dcat.mapping import Mapping
 
 from tests.conftest import clear_elastic
-from tests.app.test_app import BlueprintClient
+from tests.app.flask.test_app import BlueprintClient
 
 
 @pytest.fixture(scope='session')

tests/app/test_optimade.py

@@ -21,33 +21,11 @@ import pytest
 
 from nomad.processing import Upload
 from nomad import search
+from nomad.app.optimade import parse_filter
 
-from nomad.app.optimade import parse_filter, url
-
-from tests.app.test_app import BlueprintClient
 from tests.conftest import clear_elastic, clear_raw_files
 
 
-@pytest.fixture(scope='session')
-def api(session_client):
-    return BlueprintClient(session_client, '/optimade/v1')
-
-
-@pytest.fixture(scope='session')
-def index_api(session_client):
-    return BlueprintClient(session_client, '/optimade/index/v1')
-
-
-def test_index(index_api):
-    rv = index_api.get('/info')
-    assert rv.status_code == 200
-
-    rv = index_api.get('/links')
-    assert rv.status_code == 200
-    data = json.loads(rv.data)
-    assert data['data'][0]['attributes']['base_url']['href'].endswith('optimade')
-
-
 def test_get_entry(published: Upload):
     calc_id = list(published.calcs)[0].calc_id
     with published.upload_files.read_archive(calc_id) as archive:
@@ -58,18 +36,17 @@ def test_get_entry(published: Upload):
     assert 'dft.optimade.chemical_formula_hill' in search.flat(search_result)
 
 
-def test_no_optimade(mongo, elastic, raw_files, api):
-    from tests.app.utils import Upload
+def test_no_optimade(mongo, elastic, raw_files, client):
+    from tests.app.flask.utils import Upload
     upload = Upload()
     upload.create_test_structure(1, 2, 1, [], 0)
     upload.create_test_structure(2, 2, 1, [], 0, optimade=False)
     upload.create_upload_files()
     search.refresh()
 
-    rv = api.get('/calculations')
+    rv = client.get('/optimade/structures')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
-
+    data = rv.json()
     assert data['meta']['data_returned'] == 1
 
 
@@ -78,7 +55,7 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
     clear_elastic(elastic_infra)
     mongo_infra.drop_database('test_db')
 
-    from tests.app.utils import Upload
+    from tests.app.flask.utils import Upload
     upload = Upload()
     upload.create_test_structure(1, 2, 1, [], 0)
     upload.create_test_structure(2, 2, 1, ['C'], 0)
@@ -100,7 +77,9 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
     ('nelements < 3', 3),
     ('nelements <= 3', 4),
     ('nelements != 2', 1),
+    ('2 != nelements', 1),
     ('1 < nelements', 4),
+    ('3 <= nelements', 1),
     ('elements HAS "H"', 4),
     ('elements HAS ALL "H", "O"', 4),
     ('elements HAS ALL "H", "C"', 1),
@@ -131,7 +110,7 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
     ('NOT chemical_formula_descriptive ENDS WITH "1"', 4),
     ('chemical_formula_descriptive CONTAINS "C" AND NOT chemical_formula_descriptive STARTS WITH "O"', 1),
     ('NOT chemical_formula_anonymous STARTS WITH "A"', 0),
-    ('chemical_formula_anonymous CONTAINS "AB2" AND chemical_formula_anonymous ENDS WITH "C"', 1),
+    ('chemical_formula_anonymous CONTAINS "A2B" AND chemical_formula_anonymous ENDS WITH "C"', 1),
     ('nsites >=3 AND elements LENGTH = 2', 2),
     ('elements LENGTH = 2', 3),
     ('elements LENGTH 2', 3),
@@ -159,14 +138,10 @@ def test_optimade_parser(example_structures, query, results):
             query = parse_filter(query)
 
 
-def test_url():
-    assert url('endpoint', param='value').endswith('/optimade/v1/endpoint?param=value')
-
-
-def test_list_endpoint(api, example_structures):
-    rv = api.get('/structures')
+def test_list_endpoint(client, example_structures):
+    rv = client.get('/optimade/structures')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     for entry in ['data', 'links', 'meta']:
         assert entry in data
     assert len(data['data']) == 4
@@ -179,23 +154,23 @@ def assert_eq_attrib(data, key, ref, item=None):
         assert data['data'][item]['attributes'][key] == ref
 
 
-@pytest.mark.parametrize('limit, number, results', [
-    (1, 1, 1), (1, 5, 0), (5, 1, 4)
+@pytest.mark.parametrize('limit, offset, results', [
+    (1, 1, 1), (3, 2, 2), (5, 0, 4)
 ])
-def test_list_endpoint_pagination(api, example_structures, limit, number, results):
-    rv = api.get('/structures?page_limit=%d&page_number=%d' % (limit, number))
+def test_list_endpoint_pagination(client, example_structures, limit, offset, results):
+    rv = client.get('/optimade/structures?page_limit=%d&page_offset=%d' % (limit, offset))
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     assert len(data['data']) == results
 
 
 @pytest.mark.parametrize('sort, order', [
     ('nelements', 1), ('-nelements', -1)
 ])
-def test_list_endpoint_sort(api, example_structures, sort, order):
-    rv = api.get('/structures?sort=%s' % sort)
+def test_list_endpoint_sort(client, example_structures, sort, order):
+    rv = client.get('/optimade/structures?sort=%s' % sort)
     assert rv.status_code == 200
-    data = json.loads(rv.data)['data']
+    data = rv.json()['data']
 
     assert len(data) > 0
     for i, item in enumerate(data):
@@ -206,10 +181,10 @@ def test_list_endpoint_sort(api, example_structures, sort, order):
                 assert item['attributes']['nelements'] <= data[i - 1]['attributes']['nelements']
 
 
-def test_list_endpoint_response_fields(api, example_structures):
-    rv = api.get('/structures?response_fields=nelements,elements')
-    assert rv.status_code == 200
-    data = json.loads(rv.data)
+def test_list_endpoint_response_fields(client, example_structures):
+    rv = client.get('/optimade/structures?response_fields=nelements,elements')
+    assert rv.status_code == 200, json.dumps(rv.json(), indent=2)
+    data = rv.json()
     ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']]
     data['data'] = sorted(data['data'], key=lambda x: x['id'])
     for i in range(len(data['data'])):
@@ -219,10 +194,10 @@ def test_list_endpoint_response_fields(api, example_structures):
         assert_eq_attrib(data, 'nelements', len(ref_elements[i]), i)
 
 
-def test_single_endpoint_response_fields(api, example_structures):
-    rv = api.get('/structures/%s?response_fields=nelements,elements' % 'test_calc_id_1')
-    assert rv.status_code == 200
-    data = json.loads(rv.data)
+def test_single_endpoint_response_fields(client, example_structures):
+    rv = client.get('/optimade/structures/%s?response_fields=nelements,elements' % 'test_calc_id_1')
+    assert rv.status_code == 200, json.dumps(rv.json(), indent=2)
+    data = rv.json()
     ref_elements = ['H', 'O']
     rf = sorted(list(data['data']['attributes'].keys()))
     assert rf == ['elements', 'nelements']
@@ -230,10 +205,10 @@ def test_single_endpoint_response_fields(api, example_structures):
     assert_eq_attrib(data, 'nelements', len(ref_elements))
 
 
-def test_single_endpoint(api, example_structures):
-    rv = api.get('/structures/%s' % 'test_calc_id_1')
+def test_single_endpoint(client, example_structures):
+    rv = client.get('/optimade/structures/%s' % 'test_calc_id_1')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     for key in ['type', 'id', 'attributes']:
         assert key in data['data']
     fields = ['elements', 'nelements', 'elements_ratios',
@@ -245,39 +220,41 @@ def test_single_endpoint(api, example_structures):
         assert field in data['data']['attributes']
 
 
-def test_base_info_endpoint(api):
-    rv = api.get('/info')
+def test_base_info_endpoint(client):
+    rv = client.get('/optimade/info')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     for key in ['type', 'id', 'attributes']:
         assert key in data['data']
     assert data['data']['type'] == 'info'
     assert data['data']['id'] == '/'
 
 
-@pytest.mark.parametrize('entry_type', ['calculations', 'structures'])
-def test_entry_info_endpoint(api, entry_type):
-    rv = api.get('/info/%s' % entry_type)
+def test_entry_info_endpoint(client):
+    rv = client.get('/optimade/info/structures')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     for key in ['description', 'properties', 'formats', 'output_fields_by_format']:
         assert key in data['data']
 
-    assert '_nmd_atoms' in data['data']['properties']
-    assert '_nmd_dft_system' in data['data']['properties']
+    # TODO this does not seem to be supported by optimade-python-tools
+    # assert '_nmd_atoms' in data['data']['properties']
+    # assert '_nmd_dft_system' in data['data']['properties']
 
 
-def test_links_endpoint(api, example_structures):
-    rv = api.get('/links')
+def test_links_endpoint(client, example_structures):
+    rv = client.get('/optimade/links')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
-    assert data['data'][0]['attributes']['base_url']['href'].endswith('optimade/index')
+    data = rv.json()
+
+    nomad_link = next(link for link in data['data'] if link['id'] == 'index')
+    assert nomad_link['attributes']['base_url'].endswith('/nmd')
 
 
-def test_structures_endpoint(api, example_structures):
-    rv = api.get('/structures')
+def test_structures_endpoint(client, example_structures):
+    rv = client.get('/optimade/structures')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     assert len(data['data']) == 4
     for d in data['data']:
         for key in ['id', 'attributes']:
@@ -290,10 +267,10 @@ def test_structures_endpoint(api, example_structures):
             assert key in d['attributes']
 
 
-def test_structure_endpoint(api, example_structures):
-    rv = api.get('/structures/%s' % 'test_calc_id_1')
+def test_structure_endpoint(client, example_structures):
+    rv = client.get('/optimade/structures/%s' % 'test_calc_id_1')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     assert data.get('data') is not None
     attr = data['data'].get('attributes')
     assert attr is not None
@@ -301,33 +278,10 @@ def test_structure_endpoint(api, example_structures):
     assert len(attr.get('dimension_types')) == 3
 
 
-def test_calculations_endpoint(api, example_structures):
-    rv = api.get('/calculations')
-    assert rv.status_code == 200
-    data = json.loads(rv.data)
-    assert len(data['data']) == 4
-    for d in data['data']:
-        for key in ['id', 'attributes']:
-            assert d.get(key) is not None
-        required_keys = ['last_modified']
-        for key in required_keys:
-            assert key in d['attributes']
-
-
-def test_calculation_endpoint(api, example_structures):
-    rv = api.get('/calculations/%s' % 'test_calc_id_1')
-    assert rv.status_code == 200
-    data = json.loads(rv.data)
-    assert data.get('data') is not None
-    attr = data['data'].get('attributes')
-    assert attr is not None
-    assert len(attr) == 2
-
-
-def test_nmd_properties(api, example_structures):
-    rv = api.get('/structures/%s' % 'test_calc_id_1?response_fields=_nmd_atoms,_nmd_dft_system,_nmd_doesnotexist')
+def test_nmd_properties(client, example_structures):
+    rv = client.get('/optimade/structures/%s' % 'test_calc_id_1?response_fields=_nmd_atoms,_nmd_dft_system,_nmd_doesnotexist')
     assert rv.status_code == 200
-    data = json.loads(rv.data)
+    data = rv.json()
     assert data.get('data') is not None
     attr = data['data'].get('attributes')
     assert attr is not None

tests/app/v1/conftest.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pytest
+from fastapi.testclient import TestClient
+
+from nomad.app.main import app
+
+
+@pytest.fixture(scope='session')
+def client():
+    return TestClient(app, base_url='http://testserver/api/v1/')

tests/app/v1/routers/common.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from devtools import debug
+
+
+def assert_response(response, status_code=None):
+    ''' General assertions for status_code and error messages '''
+    if status_code and response.status_code != status_code:
+        try:
+            debug(response.json())
+        except Exception:
+            pass
+
+    if status_code is not None:
+        if response.status_code != status_code and response.status_code == 422:
+            print(response.json()['detail'])
+        assert response.status_code == status_code
+
+    if status_code == 422:
+        response_json = response.json()
+        details = response_json['detail']
+        assert len(details) > 0
+        for detail in details:
+            assert 'loc' in detail
+            assert 'msg' in detail
+        return
+
+    if 400 <= status_code < 500:
+        response_json = response.json()
+        assert 'detail' in response_json

tests/app/v1/routers/test_auth.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import pytest
+from urllib.parse import urlencode
+
+
+def perform_get_token_test(client, http_method, status_code, username, password):
+    if http_method == 'post':
+        response = client.post(
+            'auth/token',
+            data=dict(username=username, password=password))
+    else:
+        response = client.get('auth/token?%s' % urlencode(
+            dict(username=username, password=password)))
+
+    assert response.status_code == status_code
+
+
+@pytest.mark.parametrize('http_method', ['post', 'get'])
+def test_get_token(client, test_user, http_method):
+    perform_get_token_test(client, http_method, 200, test_user.username, 'password')
+
+
+@pytest.mark.parametrize('http_method', ['post', 'get'])
+def test_get_token_bad_credentials(client, http_method):
+    perform_get_token_test(client, http_method, 401, 'bad', 'credentials')

tests/app/v1/routers/test_datasets.py

+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import List
+import pytest
+from urllib.parse import urlencode
+from datetime import datetime
+
+from nomad.datamodel import Dataset
+from nomad import search, processing
+from nomad.app.v1.models import Query
+
+from tests.app.conftest import ExampleData
+from tests.conftest import admin_user_id
+
+from .test_entries import data as example_entries  # pylint: disable=unused-import
+from .common import assert_response
+
+'''
+These are the tests for all API operations below ``datasets``. The tests are organized
+using the following type of methods: fixtures, ``perfrom_*_test``, ``assert_*``, and
+``test_*``. While some ``test_*`` methods test individual API operations, some
+test methods will test multiple API operations that use common aspects like
+supporting queries, pagination, or the owner parameter. The test methods will use
+``perform_*_test`` methods as an parameter. Similarely, the ``assert_*`` methods allow
+to assert for certain aspects in the responses.
+'''
+
+
+@pytest.fixture(scope='function')
+def data(elastic, raw_files, mongo, test_user, other_test_user):
+    def create_dataset(**kwargs):
+        dataset = Dataset(created=datetime.now(), modified=datetime.now(), **kwargs)
+        dataset.m_get_annotations('mongo').save()
+        return dataset
+
+    data = ExampleData(uploader=test_user)
+    data._create_entry(
+        upload_id='upload_1',
+        calc_id='entry_1',
+        mainfile='test_content/1/mainfile.json',
+        datasets=[
+            create_dataset(
+                dataset_id='dataset_1',
+                user_id=test_user.user_id,
+                name='test dataset 1',
+                dataset_type='owned'),
+            create_dataset(
+                dataset_id='dataset_2',
+                user_id=test_user.user_id,
+                name='test dataset 2',
+                dataset_type='owned')
+        ])
+
+    data._create_entry(
+        upload_id='upload_1',
+        calc_id='entry_2',
+        mainfile='test_content/2/mainfile.json',
+        datasets=[
+            create_dataset(
+                dataset_id='dataset_listed',
+                user_id=test_user.user_id,
+                name='foreign test dataset',
+                dataset_type='foreign'),
+            create_dataset(