Commit 873efcae authored by Markus Scheidgen's avatar Markus Scheidgen
Browse files

Merge branch 'v0.10.0' into 'master'

Merge v0.10.0 into master for release

Closes #475, #484, #497, #492, #498, and #500

See merge request !283
parents 5636e115 015742db
Pipeline #95934 passed with stage
in 2 minutes and 28 seconds
......@@ -30,22 +30,23 @@ import base64
import itertools
from hashlib import md5
from nomad.app.common import rfc3339DateTime
from nomad.app.api.auth import generate_upload_token
from nomad.app.flask.common import rfc3339DateTime
from nomad.app.flask.api.auth import generate_upload_token
from nomad import search, files, config, utils, infrastructure
from nomad.metainfo import search_extension
from nomad.files import UploadFiles, PublicUploadFiles
from nomad.processing import Upload, Calc, SUCCESS
from nomad.datamodel import EntryMetadata, User, Dataset
from tests.conftest import create_auth_headers, clear_elastic, clear_raw_files
from tests.test_files import example_file, example_file_mainfile, example_file_contents
from tests.test_files import create_staging_upload, create_public_upload, assert_upload_files
from tests.test_search import assert_search_upload
from tests.processing import test_data as test_processing
from tests.processing.test_data import oasis_publishable_upload
from tests.processing.test_data import oasis_publishable_upload # pylint: disable=unused-import
from tests.conftest import clear_elastic, clear_raw_files
from tests.app.test_app import BlueprintClient
from ..conftest import create_auth_headers
from .test_app import BlueprintClient
logger = utils.get_logger(__name__)
......@@ -1033,6 +1034,12 @@ class TestRepo():
assert data['code']['python'] is not None
assert data['code']['clientlib'] is not None
def test_get_defaults(self, api, example_elastic_calcs, no_warn, test_user_auth):
rv = api.get('/repo/0/1', headers=test_user_auth)
assert rv.status_code == 200
data = rv.json
assert data['license'] is not None
def test_public_calc(self, api, example_elastic_calcs, no_warn, other_test_user_auth):
rv = api.get('/repo/0/1', headers=other_test_user_auth)
assert rv.status_code == 200
......@@ -1169,7 +1176,7 @@ class TestRepo():
assert value in statistics['dft.system']
def test_search_exclude(self, api, example_elastic_calcs, no_warn):
rv = api.get('/repo/?exclude=atoms,only_atoms')
rv = api.get('/repo/?exclude=atoms,only_atoms,dft.optimade,dft.quantities')
assert rv.status_code == 200
result = search.flat(json.loads(rv.data)['results'][0])
assert 'atoms' not in result
......@@ -1513,7 +1520,7 @@ class TestEditRepo():
@pytest.fixture(autouse=True)
def example_data(self, class_api, test_user, other_test_user, raw_files):
from tests.app.utils import Upload
from .utils import Upload
uploads = {}
for i in range(1, 4):
......
......@@ -25,7 +25,7 @@ from nomad import config
from nomad.cli import cli
from nomad import processing as proc, infrastructure
from tests.app.test_app import BlueprintClient
from tests.app.flask.test_app import BlueprintClient
silicon_id = "fh3UBjhUVm4nxzeRd2JJuqw5oXYa"
......
......@@ -21,10 +21,8 @@
import json
import pytest
from nomad import config
from tests.utils import assert_log
from tests.app import resource # pylint: disable=unused-import
from . import resource # pylint: disable=unused-import
class BlueprintClient():
......@@ -82,7 +80,7 @@ def test_internal_server_error_post(client, caplog):
assert data['json']['test_arg'] == 'value'
@pytest.mark.parametrize('api', ['api', 'optimade'])
@pytest.mark.parametrize('api', ['api'])
def test_swagger(client, api):
rv = client.get('/%s/swagger.json' % api)
assert rv.status_code == 200
......
......@@ -21,10 +21,10 @@ from datetime import datetime
from nomad import infrastructure, config
from nomad.datamodel import EntryMetadata
from nomad.app.dcat.mapping import Mapping
from nomad.app.flask.dcat.mapping import Mapping
from tests.conftest import clear_elastic
from tests.app.test_app import BlueprintClient
from tests.app.flask.test_app import BlueprintClient
@pytest.fixture(scope='session')
......
......@@ -21,33 +21,11 @@ import pytest
from nomad.processing import Upload
from nomad import search
from nomad.app.optimade import parse_filter
from nomad.app.optimade import parse_filter, url
from tests.app.test_app import BlueprintClient
from tests.conftest import clear_elastic, clear_raw_files
@pytest.fixture(scope='session')
def api(session_client):
return BlueprintClient(session_client, '/optimade/v1')
@pytest.fixture(scope='session')
def index_api(session_client):
return BlueprintClient(session_client, '/optimade/index/v1')
def test_index(index_api):
rv = index_api.get('/info')
assert rv.status_code == 200
rv = index_api.get('/links')
assert rv.status_code == 200
data = json.loads(rv.data)
assert data['data'][0]['attributes']['base_url']['href'].endswith('optimade')
def test_get_entry(published: Upload):
calc_id = list(published.calcs)[0].calc_id
with published.upload_files.read_archive(calc_id) as archive:
......@@ -58,18 +36,17 @@ def test_get_entry(published: Upload):
assert 'dft.optimade.chemical_formula_hill' in search.flat(search_result)
def test_no_optimade(mongo, elastic, raw_files, api):
from tests.app.utils import Upload
def test_no_optimade(mongo, elastic, raw_files, client):
from tests.app.flask.utils import Upload
upload = Upload()
upload.create_test_structure(1, 2, 1, [], 0)
upload.create_test_structure(2, 2, 1, [], 0, optimade=False)
upload.create_upload_files()
search.refresh()
rv = api.get('/calculations')
rv = client.get('/optimade/structures')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
assert data['meta']['data_returned'] == 1
......@@ -78,7 +55,7 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
clear_elastic(elastic_infra)
mongo_infra.drop_database('test_db')
from tests.app.utils import Upload
from tests.app.flask.utils import Upload
upload = Upload()
upload.create_test_structure(1, 2, 1, [], 0)
upload.create_test_structure(2, 2, 1, ['C'], 0)
......@@ -100,7 +77,9 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
('nelements < 3', 3),
('nelements <= 3', 4),
('nelements != 2', 1),
('2 != nelements', 1),
('1 < nelements', 4),
('3 <= nelements', 1),
('elements HAS "H"', 4),
('elements HAS ALL "H", "O"', 4),
('elements HAS ALL "H", "C"', 1),
......@@ -131,7 +110,7 @@ def example_structures(elastic_infra, mongo_infra, raw_files_infra):
('NOT chemical_formula_descriptive ENDS WITH "1"', 4),
('chemical_formula_descriptive CONTAINS "C" AND NOT chemical_formula_descriptive STARTS WITH "O"', 1),
('NOT chemical_formula_anonymous STARTS WITH "A"', 0),
('chemical_formula_anonymous CONTAINS "AB2" AND chemical_formula_anonymous ENDS WITH "C"', 1),
('chemical_formula_anonymous CONTAINS "A2B" AND chemical_formula_anonymous ENDS WITH "C"', 1),
('nsites >=3 AND elements LENGTH = 2', 2),
('elements LENGTH = 2', 3),
('elements LENGTH 2', 3),
......@@ -159,14 +138,10 @@ def test_optimade_parser(example_structures, query, results):
query = parse_filter(query)
def test_url():
assert url('endpoint', param='value').endswith('/optimade/v1/endpoint?param=value')
def test_list_endpoint(api, example_structures):
rv = api.get('/structures')
def test_list_endpoint(client, example_structures):
rv = client.get('/optimade/structures')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
for entry in ['data', 'links', 'meta']:
assert entry in data
assert len(data['data']) == 4
......@@ -179,23 +154,23 @@ def assert_eq_attrib(data, key, ref, item=None):
assert data['data'][item]['attributes'][key] == ref
@pytest.mark.parametrize('limit, number, results', [
(1, 1, 1), (1, 5, 0), (5, 1, 4)
@pytest.mark.parametrize('limit, offset, results', [
(1, 1, 1), (3, 2, 2), (5, 0, 4)
])
def test_list_endpoint_pagination(api, example_structures, limit, number, results):
rv = api.get('/structures?page_limit=%d&page_number=%d' % (limit, number))
def test_list_endpoint_pagination(client, example_structures, limit, offset, results):
rv = client.get('/optimade/structures?page_limit=%d&page_offset=%d' % (limit, offset))
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
assert len(data['data']) == results
@pytest.mark.parametrize('sort, order', [
('nelements', 1), ('-nelements', -1)
])
def test_list_endpoint_sort(api, example_structures, sort, order):
rv = api.get('/structures?sort=%s' % sort)
def test_list_endpoint_sort(client, example_structures, sort, order):
rv = client.get('/optimade/structures?sort=%s' % sort)
assert rv.status_code == 200
data = json.loads(rv.data)['data']
data = rv.json()['data']
assert len(data) > 0
for i, item in enumerate(data):
......@@ -206,10 +181,10 @@ def test_list_endpoint_sort(api, example_structures, sort, order):
assert item['attributes']['nelements'] <= data[i - 1]['attributes']['nelements']
def test_list_endpoint_response_fields(api, example_structures):
rv = api.get('/structures?response_fields=nelements,elements')
assert rv.status_code == 200
data = json.loads(rv.data)
def test_list_endpoint_response_fields(client, example_structures):
rv = client.get('/optimade/structures?response_fields=nelements,elements')
assert rv.status_code == 200, json.dumps(rv.json(), indent=2)
data = rv.json()
ref_elements = [['H', 'O'], ['C', 'H', 'O'], ['H', 'O'], ['H', 'O']]
data['data'] = sorted(data['data'], key=lambda x: x['id'])
for i in range(len(data['data'])):
......@@ -219,10 +194,10 @@ def test_list_endpoint_response_fields(api, example_structures):
assert_eq_attrib(data, 'nelements', len(ref_elements[i]), i)
def test_single_endpoint_response_fields(api, example_structures):
rv = api.get('/structures/%s?response_fields=nelements,elements' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
def test_single_endpoint_response_fields(client, example_structures):
rv = client.get('/optimade/structures/%s?response_fields=nelements,elements' % 'test_calc_id_1')
assert rv.status_code == 200, json.dumps(rv.json(), indent=2)
data = rv.json()
ref_elements = ['H', 'O']
rf = sorted(list(data['data']['attributes'].keys()))
assert rf == ['elements', 'nelements']
......@@ -230,10 +205,10 @@ def test_single_endpoint_response_fields(api, example_structures):
assert_eq_attrib(data, 'nelements', len(ref_elements))
def test_single_endpoint(api, example_structures):
rv = api.get('/structures/%s' % 'test_calc_id_1')
def test_single_endpoint(client, example_structures):
rv = client.get('/optimade/structures/%s' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
for key in ['type', 'id', 'attributes']:
assert key in data['data']
fields = ['elements', 'nelements', 'elements_ratios',
......@@ -245,39 +220,41 @@ def test_single_endpoint(api, example_structures):
assert field in data['data']['attributes']
def test_base_info_endpoint(api):
rv = api.get('/info')
def test_base_info_endpoint(client):
rv = client.get('/optimade/info')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
for key in ['type', 'id', 'attributes']:
assert key in data['data']
assert data['data']['type'] == 'info'
assert data['data']['id'] == '/'
@pytest.mark.parametrize('entry_type', ['calculations', 'structures'])
def test_entry_info_endpoint(api, entry_type):
rv = api.get('/info/%s' % entry_type)
def test_entry_info_endpoint(client):
rv = client.get('/optimade/info/structures')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
for key in ['description', 'properties', 'formats', 'output_fields_by_format']:
assert key in data['data']
assert '_nmd_atoms' in data['data']['properties']
assert '_nmd_dft_system' in data['data']['properties']
# TODO this does not seem to be supported by optimade-python-tools
# assert '_nmd_atoms' in data['data']['properties']
# assert '_nmd_dft_system' in data['data']['properties']
def test_links_endpoint(api, example_structures):
rv = api.get('/links')
def test_links_endpoint(client, example_structures):
rv = client.get('/optimade/links')
assert rv.status_code == 200
data = json.loads(rv.data)
assert data['data'][0]['attributes']['base_url']['href'].endswith('optimade/index')
data = rv.json()
nomad_link = next(link for link in data['data'] if link['id'] == 'index')
assert nomad_link['attributes']['base_url'].endswith('/nmd')
def test_structures_endpoint(api, example_structures):
rv = api.get('/structures')
def test_structures_endpoint(client, example_structures):
rv = client.get('/optimade/structures')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
assert len(data['data']) == 4
for d in data['data']:
for key in ['id', 'attributes']:
......@@ -290,10 +267,10 @@ def test_structures_endpoint(api, example_structures):
assert key in d['attributes']
def test_structure_endpoint(api, example_structures):
rv = api.get('/structures/%s' % 'test_calc_id_1')
def test_structure_endpoint(client, example_structures):
rv = client.get('/optimade/structures/%s' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
assert data.get('data') is not None
attr = data['data'].get('attributes')
assert attr is not None
......@@ -301,33 +278,10 @@ def test_structure_endpoint(api, example_structures):
assert len(attr.get('dimension_types')) == 3
def test_calculations_endpoint(api, example_structures):
rv = api.get('/calculations')
assert rv.status_code == 200
data = json.loads(rv.data)
assert len(data['data']) == 4
for d in data['data']:
for key in ['id', 'attributes']:
assert d.get(key) is not None
required_keys = ['last_modified']
for key in required_keys:
assert key in d['attributes']
def test_calculation_endpoint(api, example_structures):
rv = api.get('/calculations/%s' % 'test_calc_id_1')
assert rv.status_code == 200
data = json.loads(rv.data)
assert data.get('data') is not None
attr = data['data'].get('attributes')
assert attr is not None
assert len(attr) == 2
def test_nmd_properties(api, example_structures):
rv = api.get('/structures/%s' % 'test_calc_id_1?response_fields=_nmd_atoms,_nmd_dft_system,_nmd_doesnotexist')
def test_nmd_properties(client, example_structures):
rv = client.get('/optimade/structures/%s' % 'test_calc_id_1?response_fields=_nmd_atoms,_nmd_dft_system,_nmd_doesnotexist')
assert rv.status_code == 200
data = json.loads(rv.data)
data = rv.json()
assert data.get('data') is not None
attr = data['data'].get('attributes')
assert attr is not None
......
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from fastapi.testclient import TestClient
from nomad.app.main import app
@pytest.fixture(scope='session')
def client():
return TestClient(app, base_url='http://testserver/api/v1/')
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from devtools import debug
def assert_response(response, status_code=None):
''' General assertions for status_code and error messages '''
if status_code and response.status_code != status_code:
try:
debug(response.json())
except Exception:
pass
if status_code is not None:
if response.status_code != status_code and response.status_code == 422:
print(response.json()['detail'])
assert response.status_code == status_code
if status_code == 422:
response_json = response.json()
details = response_json['detail']
assert len(details) > 0
for detail in details:
assert 'loc' in detail
assert 'msg' in detail
return
if 400 <= status_code < 500:
response_json = response.json()
assert 'detail' in response_json
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
from urllib.parse import urlencode
def perform_get_token_test(client, http_method, status_code, username, password):
if http_method == 'post':
response = client.post(
'auth/token',
data=dict(username=username, password=password))
else:
response = client.get('auth/token?%s' % urlencode(
dict(username=username, password=password)))
assert response.status_code == status_code
@pytest.mark.parametrize('http_method', ['post', 'get'])
def test_get_token(client, test_user, http_method):
perform_get_token_test(client, http_method, 200, test_user.username, 'password')
@pytest.mark.parametrize('http_method', ['post', 'get'])
def test_get_token_bad_credentials(client, http_method):
perform_get_token_test(client, http_method, 401, 'bad', 'credentials')
#
# Copyright The NOMAD Authors.
#
# This file is part of NOMAD. See https://nomad-lab.eu for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from typing import List
import pytest
from urllib.parse import urlencode
from datetime import datetime
from nomad.datamodel import Dataset
from nomad import search, processing
from nomad.app.v1.models import Query
from tests.app.conftest import ExampleData
from tests.conftest import admin_user_id
from .test_entries import data as example_entries # pylint: disable=unused-import
from .common import assert_response
'''
These are the tests for all API operations below ``datasets``. The tests are organized
using the following type of methods: fixtures, ``perfrom_*_test``, ``assert_*``, and
``test_*``. While some ``test_*`` methods test individual API operations, some
test methods will test multiple API operations that use common aspects like
supporting queries, pagination, or the owner parameter. The test methods will use
``perform_*_test`` methods as an parameter. Similarely, the ``assert_*`` methods allow
to assert for certain aspects in the responses.
'''
@pytest.fixture(scope='function')
def data(elastic, raw_files, mongo, test_user, other_test_user):
def create_dataset(**kwargs):
dataset = Dataset(created=datetime.now(), modified=datetime.now(), **kwargs)
dataset.m_get_annotations('mongo').save()
return dataset
data = ExampleData(uploader=test_user)
data._create_entry(
upload_id='upload_1',
calc_id='entry_1',
mainfile='test_content/1/mainfile.json',
datasets=[
create_dataset(
dataset_id='dataset_1',
user_id=test_user.user_id,
name='test dataset 1',
dataset_type='owned'),
create_dataset(
dataset_id='dataset_2',
user_id=test_user.user_id,
name='test dataset 2',
dataset_type='owned')
])
data._create_entry(
upload_id='upload_1',
calc_id='entry_2',
mainfile='test_content/2/mainfile.json',
datasets=[
create_dataset(
dataset_id='dataset_listed',
user_id=test_user.user_id,
name='foreign test dataset',
dataset_type='foreign'),
create_dataset(