diff --git a/nomad/app/main.py b/nomad/app/main.py index 7a29286dd920000851656c15b191e4abd8ed916b..3bdceecf16608e27adf3bcdaee4131bb14cd0dd1 100644 --- a/nomad/app/main.py +++ b/nomad/app/main.py @@ -16,110 +16,22 @@ # limitations under the License. # -from fastapi import FastAPI, status, Request -from fastapi.responses import JSONResponse, RedirectResponse +from fastapi import FastAPI from fastapi.middleware.wsgi import WSGIMiddleware -import traceback -from nomad import config, utils +from nomad import config -from .flask import app as flask_app -from .routers import users, entries, auth, datasets from .optimade import optimade_app +from .flask import app as flask_app +from .v1.main import app as v1_app -logger = utils.get_logger(__name__) - -app = FastAPI( - openapi_url='%s/api/v1/openapi.json' % config.services.api_base_path, - docs_url='%s/api/v1/extensions/docs' % config.services.api_base_path, - redoc_url='%s/api/v1/extensions/redoc' % config.services.api_base_path, - swagger_ui_oauth2_redirect_url='%s/api/v1/docs/oauth2-redirect' % config.services.api_base_path, - - title='NOMAD API', - version='v1, NOMAD %s@%s' % (config.meta.version, config.meta.commit), - description=utils.strip(''' - **Disclaimer!** This is the new NOMAD API. It is still under development and only includes a - part of the NOMAD API functionality. You can still use the old flask-based API - as `/api` and the optimade API as `/optimade/v1`. - - ## Getting started - - ... TODO put the examples and tutorial here ... - - ## Conventions - - ### Paths - - The various API operations are organized with the following path scheme. The first - part of the path, describes the data entity that is covered by - the operations below (e.g. `entries`, `users`, `datasets`, `uploads`). For example - everything below `entries` will be about searching entries, getting - an entry, editing entries, etc. - - The second (optional and variable) path segment allows to denote a specific entity instance, - e.g. a specific entry or dataset, usually by id. With out such a variable second - path segment, its about all instances, e.g. searching entries or listing all datasets. - - Optional (if available) further path segments will determine the variety and format - of data. This is mostly for entries to distinguish the metadata, raw, and archive - data or distinguish between listing (i.e. paginated json) and downloading - (i.e. streaming a zip-file) - - Further, we try to adhere to the paradim of getting and posting resources. Therefore, - when you post a complex query, you will not post it to `/entries` (a query is not an entry), - but `/entries/query`. Here *query* being a kind of virtual resource. - - ### Parameters and bodies for GET and POST operations - - We offer **GET** and **POST** versions for many complex operations. The idea is that - **GET** is easy to use, e.g. via curl or simply in the browser, while **POST** - allows to provide more complex parameters (i.e. a JSON body). For example to - search for entries, you can use the **GET** operation `/entries` to specify simple - queries via URL, e.g. `/entries?code_name=VASP&atoms=Ti`, but you would use - **POST** `/entries/query` to provide a complex nested queries, e.g. with logical - operators. - - Typicall the **POST** version is a super-set of the functionality of the **GET** - version. But, most top-level parameters in the **POST** body, will be available - in the **GET** version as URL parameters with the same name and meaning. This - is especially true for reoccuring parameters for general API concepts like pagination - or specifying required result fields. - - ### Response layout - - Typically a response will mirror all input parameters in the normalized form that - was used to perform the operation. - - Some of these will be augmented with result values. For example the pagination - section of a request will be augmented with the total available number. - - The actual requested data, will be placed under the key `data`. - - ## About Authentication - - NOMAD is an open datasharing platform, and most of the API operations do not require - any authorization and can be freely used without a user or credentials. However, - to upload data, edit data, or view your own and potentially unpublished data, - the API needs to authenticate you. - - The NOMAD API uses OAuth and tokens to authenticate users. We provide simple operations - that allow you to acquire an *access token* via username and password based - authentication (`/auth/token`). The resulting access token can then be used on all operations - (e.g. that support or require authentication). - - To use authentication in the dashboard, simply use the Authorize button. The - dashboard GUI will manage the access token and use it while you try out the various - operations. - ''')) - - -async def redirect_to_docs(req: Request): - return RedirectResponse('%s/api/v1/extensions/docs' % config.services.api_base_path) - +app = FastAPI() -app.add_route('%s/api/v1' % config.services.api_base_path, redirect_to_docs, include_in_schema=False) -app.add_route('%s/api/v1/' % config.services.api_base_path, redirect_to_docs, include_in_schema=False) +app_base = config.services.api_base_path +app.mount(f'{app_base}/api/v1', v1_app) +app.mount(f'{app_base}/optimade', optimade_app) +app.mount(app_base, WSGIMiddleware(flask_app)) @app.on_event('startup') @@ -133,27 +45,3 @@ async def startup_event(): pass infrastructure.setup() - - -@app.exception_handler(Exception) -async def unicorn_exception_handler(request: Request, e: Exception): - logger.error('unexpected exception in API', url=request.url, exc_info=e) - return JSONResponse( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - content={ - 'detail': { - 'reason': 'Unexpected exception while handling your request', - 'exception': str(e), - 'exception_class': e.__class__.__name__, - 'exception_traceback': traceback.format_exc() - } - } - ) - -app.include_router(auth.router, prefix='%s/api/v1/auth' % config.services.api_base_path) -app.include_router(users.router, prefix='%s/api/v1/users' % config.services.api_base_path) -app.include_router(entries.router, prefix='%s/api/v1/entries' % config.services.api_base_path) -app.include_router(datasets.router, prefix='%s/api/v1/datasets' % config.services.api_base_path) - -app.mount('%s/optimade' % config.services.api_base_path, optimade_app) -app.mount(config.services.api_base_path, WSGIMiddleware(flask_app)) diff --git a/nomad/app/routers/__init__.py b/nomad/app/v1/__init__.py similarity index 100% rename from nomad/app/routers/__init__.py rename to nomad/app/v1/__init__.py diff --git a/nomad/app/v1/common.py b/nomad/app/v1/common.py new file mode 100644 index 0000000000000000000000000000000000000000..5a438236be3b9fe19862d8f45fd39bcee59403f8 --- /dev/null +++ b/nomad/app/v1/common.py @@ -0,0 +1,21 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from nomad import config + +root_path = f'{config.services.api_base_path}/api/v1' diff --git a/nomad/app/v1/main.py b/nomad/app/v1/main.py new file mode 100644 index 0000000000000000000000000000000000000000..3f3c757492a1541308ccae35820dacd2c0190d50 --- /dev/null +++ b/nomad/app/v1/main.py @@ -0,0 +1,154 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from fastapi import FastAPI, status, Request +from fastapi.responses import JSONResponse, RedirectResponse +import traceback + +from nomad import config, utils + +from .common import root_path +from .routers import users, entries, auth, datasets + + +logger = utils.get_logger(__name__) + +app = FastAPI( + root_path=root_path, + openapi_url='/openapi.json', + docs_url='/extensions/docs', + redoc_url='/extensions/redoc', + swagger_ui_oauth2_redirect_url='/extensions/docs/oauth2-redirect', + title='NOMAD API', + version='v1, NOMAD %s@%s' % (config.meta.version, config.meta.commit), + description=utils.strip(''' + **Disclaimer!** This is the new NOMAD API. It is still under development and only includes a + part of the NOMAD API functionality. You can still use the old flask-based API + as `/api` and the optimade API as `/optimade/v1`. + + ## Getting started + + ... TODO put the examples and tutorial here ... + + ## Conventions + + ### Paths + + The various API operations are organized with the following path scheme. The first + part of the path, describes the data entity that is covered by + the operations below (e.g. `entries`, `users`, `datasets`, `uploads`). For example + everything below `entries` will be about searching entries, getting + an entry, editing entries, etc. + + The second (optional and variable) path segment allows to denote a specific entity instance, + e.g. a specific entry or dataset, usually by id. With out such a variable second + path segment, its about all instances, e.g. searching entries or listing all datasets. + + Optional (if available) further path segments will determine the variety and format + of data. This is mostly for entries to distinguish the metadata, raw, and archive + data or distinguish between listing (i.e. paginated json) and downloading + (i.e. streaming a zip-file) + + Further, we try to adhere to the paradim of getting and posting resources. Therefore, + when you post a complex query, you will not post it to `/entries` (a query is not an entry), + but `/entries/query`. Here *query* being a kind of virtual resource. + + ### Parameters and bodies for GET and POST operations + + We offer **GET** and **POST** versions for many complex operations. The idea is that + **GET** is easy to use, e.g. via curl or simply in the browser, while **POST** + allows to provide more complex parameters (i.e. a JSON body). For example to + search for entries, you can use the **GET** operation `/entries` to specify simple + queries via URL, e.g. `/entries?code_name=VASP&atoms=Ti`, but you would use + **POST** `/entries/query` to provide a complex nested queries, e.g. with logical + operators. + + Typicall the **POST** version is a super-set of the functionality of the **GET** + version. But, most top-level parameters in the **POST** body, will be available + in the **GET** version as URL parameters with the same name and meaning. This + is especially true for reoccuring parameters for general API concepts like pagination + or specifying required result fields. + + ### Response layout + + Typically a response will mirror all input parameters in the normalized form that + was used to perform the operation. + + Some of these will be augmented with result values. For example the pagination + section of a request will be augmented with the total available number. + + The actual requested data, will be placed under the key `data`. + + ## About Authentication + + NOMAD is an open datasharing platform, and most of the API operations do not require + any authorization and can be freely used without a user or credentials. However, + to upload data, edit data, or view your own and potentially unpublished data, + the API needs to authenticate you. + + The NOMAD API uses OAuth and tokens to authenticate users. We provide simple operations + that allow you to acquire an *access token* via username and password based + authentication (`/auth/token`). The resulting access token can then be used on all operations + (e.g. that support or require authentication). + + To use authentication in the dashboard, simply use the Authorize button. The + dashboard GUI will manage the access token and use it while you try out the various + operations. + ''')) + + +async def redirect_to_docs(req: Request): + return RedirectResponse(f'{root_path}/extensions/docs') + + +# app.add_route(f'{root_path}', redirect_to_docs, include_in_schema=False) +app.add_route('/', redirect_to_docs, include_in_schema=False) + + +@app.on_event('startup') +async def startup_event(): + from nomad import infrastructure + # each subprocess is supposed disconnect connect again: https://jira.mongodb.org/browse/PYTHON-2090 + try: + from mongoengine import disconnect + disconnect() + except Exception: + pass + + infrastructure.setup() + + +@app.exception_handler(Exception) +async def unicorn_exception_handler(request: Request, e: Exception): + logger.error('unexpected exception in API', url=request.url, exc_info=e) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={ + 'detail': { + 'reason': 'Unexpected exception while handling your request', + 'exception': str(e), + 'exception_class': e.__class__.__name__, + 'exception_traceback': traceback.format_exc() + } + } + ) + +app.include_router(auth.router, prefix='/auth') +app.include_router(users.router, prefix='/users') +app.include_router(entries.router, prefix='/entries') +app.include_router(datasets.router, prefix='/datasets') diff --git a/nomad/app/models.py b/nomad/app/v1/models.py similarity index 99% rename from nomad/app/models.py rename to nomad/app/v1/models.py index f7dc7c31ac69ba97b2abc5297b50845d1daaf727..1688a6d1ce798acfaa032ac6f5f4f9e814225225 100644 --- a/nomad/app/models.py +++ b/nomad/app/v1/models.py @@ -29,9 +29,10 @@ import fnmatch from nomad import datamodel # pylint: disable=unused-import from nomad.utils import strip from nomad.metainfo import Datetime, MEnum -from nomad.app.utils import parameter_dependency_from_model from nomad.metainfo.search_extension import metrics, search_quantities +from .utils import parameter_dependency_from_model + User = datamodel.User.m_def.a_pydantic.model diff --git a/tests/app/routers/__init__.py b/nomad/app/v1/routers/__init__.py similarity index 100% rename from tests/app/routers/__init__.py rename to nomad/app/v1/routers/__init__.py diff --git a/nomad/app/routers/auth.py b/nomad/app/v1/routers/auth.py similarity index 95% rename from nomad/app/routers/auth.py rename to nomad/app/v1/routers/auth.py index 0a7bcb78092311ba8c93c1fb463ae68d23733b60..c7fb6b2ffec1231679660fde14b4afb0559931f6 100644 --- a/nomad/app/routers/auth.py +++ b/nomad/app/v1/routers/auth.py @@ -22,8 +22,10 @@ from pydantic import BaseModel from nomad import infrastructure from nomad.utils import get_logger, strip -from nomad.app.models import User, HTTPExceptionModel -from nomad.app.utils import create_responses + +from ..common import root_path +from ..models import User, HTTPExceptionModel +from ..utils import create_responses logger = get_logger(__name__) @@ -36,7 +38,7 @@ class Token(BaseModel): token_type: str -oauth2_scheme = OAuth2PasswordBearer(tokenUrl='/api/v1/auth/token', auto_error=False) +oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f'{root_path}/auth/token', auto_error=False) async def get_optional_user(access_token: str = Depends(oauth2_scheme)) -> User: diff --git a/nomad/app/routers/datasets.py b/nomad/app/v1/routers/datasets.py similarity index 98% rename from nomad/app/routers/datasets.py rename to nomad/app/v1/routers/datasets.py index c5099530db4336b344505eb8e32e45c674fc33e5..0b89dec94ac63c06469ceabf988f3a051e7ed307 100644 --- a/nomad/app/routers/datasets.py +++ b/nomad/app/v1/routers/datasets.py @@ -27,13 +27,12 @@ from nomad.utils import strip, create_uuid from nomad.datamodel import Dataset as DatasetDefinitionCls from nomad.doi import DOI -from nomad.app.routers.auth import get_required_user -from nomad.app.utils import create_responses -from nomad.app.models import ( - pagination_parameters, Pagination, PaginationResponse, Query, - HTTPExceptionModel, User, Direction, Owner) - +from .auth import get_required_user from .entries import _do_exaustive_search +from ..utils import create_responses +from ..models import ( + pagination_parameters, Pagination, PaginationResponse, Query, HTTPExceptionModel, + User, Direction, Owner) router = APIRouter() diff --git a/nomad/app/routers/entries.py b/nomad/app/v1/routers/entries.py similarity index 99% rename from nomad/app/routers/entries.py rename to nomad/app/v1/routers/entries.py index c871461a57e34a368a7b5fe5abe772c9b327d030..2f5fdb03a8216effdc0a4fd65ee0a790cfeafb61 100644 --- a/nomad/app/routers/entries.py +++ b/nomad/app/v1/routers/entries.py @@ -29,9 +29,10 @@ from nomad.utils import strip from nomad.archive import ( query_archive, ArchiveQueryError, compute_required_with_referenced, read_partial_archives_from_mongo, filter_archive) -from nomad.app.utils import create_streamed_zipfile, File, create_responses -from nomad.app.routers.auth import get_optional_user -from nomad.app.models import ( + +from .auth import get_optional_user +from ..utils import create_streamed_zipfile, File, create_responses +from ..models import ( Pagination, WithQuery, MetadataRequired, EntriesMetadataResponse, EntriesMetadata, EntryMetadataResponse, query_parameters, metadata_required_parameters, Files, Query, pagination_parameters, files_parameters, User, Owner, HTTPExceptionModel, EntriesRaw, diff --git a/nomad/app/routers/users.py b/nomad/app/v1/routers/users.py similarity index 90% rename from nomad/app/routers/users.py rename to nomad/app/v1/routers/users.py index cce273ebed4b1cc443a8b0d41987a542032c37ae..399e4b0576c499edffda98189363bb423440d624 100644 --- a/nomad/app/routers/users.py +++ b/nomad/app/v1/routers/users.py @@ -18,11 +18,12 @@ from fastapi import Depends, APIRouter, status -from nomad.app.routers.auth import get_required_user -from nomad.app.models import User, HTTPExceptionModel -from nomad.app.utils import create_responses from nomad.utils import strip +from .auth import get_required_user +from ..models import User, HTTPExceptionModel +from ..utils import create_responses + router = APIRouter() default_tag = 'users' diff --git a/nomad/app/utils.py b/nomad/app/v1/utils.py similarity index 100% rename from nomad/app/utils.py rename to nomad/app/v1/utils.py diff --git a/nomad/search.py b/nomad/search.py index 7ee0780eb84646092fcaf9956fbce03b39cf4d85..c25594f05491c3c095cfe21ed9ccee3fea2b55af 100644 --- a/nomad/search.py +++ b/nomad/search.py @@ -31,8 +31,8 @@ from nomad.datamodel.material import Material from nomad import config, datamodel, infrastructure, utils from nomad.metainfo.search_extension import ( # pylint: disable=unused-import search_quantities, metrics, order_default_quantities, groups) -from nomad.app import models as api_models -from nomad.app.models import ( +from nomad.app.v1 import models as api_models +from nomad.app.v1.models import ( Pagination, PaginationResponse, Query, MetadataRequired, SearchResponse, Aggregation, Statistic, StatisticResponse, AggregationOrderType, AggregationResponse, AggregationDataItem) diff --git a/tests/app/conftest.py b/tests/app/conftest.py index e6547afaaf646f6fa855c8d0968d4010521abcef..dfb049032de58fe20fd83e440f1e6956f1511017 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -50,7 +50,7 @@ def admin_user_auth(admin_user: User): @pytest.fixture(scope='session') def client(): - return TestClient(app, base_url='http://testserver/api/v1/') + return TestClient(app, base_url='http://testserver/') @pytest.fixture(scope='module') diff --git a/tests/app/test_utils.py b/tests/app/test_utils.py deleted file mode 100644 index 59c6c69020c9f7bad76258c9854c1fc20757780f..0000000000000000000000000000000000000000 --- a/tests/app/test_utils.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright The NOMAD Authors. -# -# This file is part of NOMAD. See https://nomad-lab.eu for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from typing import Iterator -import os.path -import zipfile - -from nomad import config -from nomad.datamodel import EntryArchive, EntryMetadata -from nomad.app.utils import create_streamed_zipfile, File - -from tests.conftest import clear_raw_files -from tests.test_files import create_test_upload_files - - -def test_create_streamed_zip(raw_files_infra): - # We use the files of a simpe test upload to create streamed zip with all the raw - # files. - archive = EntryArchive() - metadata = archive.m_create(EntryMetadata) - metadata.upload_id = 'test_id' - metadata.calc_id = 'test_id' - metadata.mainfile = 'root/subdir/mainfile.json' - - upload_files = create_test_upload_files('test_id', [archive]) - - def generate_files() -> Iterator[File]: - for path in upload_files.raw_file_manifest(): - with upload_files.raw_file(path) as f: - yield File( - path=path, - f=f, - size=upload_files.raw_file_size(path)) - - if not os.path.exists(config.fs.tmp): - os.makedirs(config.fs.tmp) - - zip_file_path = os.path.join(config.fs.tmp, 'results.zip') - with open(zip_file_path, 'wb') as f: - for content in create_streamed_zipfile(generate_files()): - f.write(content) - - with zipfile.ZipFile(zip_file_path) as zf: - assert zf.testzip() is None - - clear_raw_files() diff --git a/tests/app/v1/__init__.py b/tests/app/v1/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/app/v1/conftest.py b/tests/app/v1/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..116ba3d1d2a900b1dcdffb06d2171e52b269cd33 --- /dev/null +++ b/tests/app/v1/conftest.py @@ -0,0 +1,28 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import pytest +from fastapi.testclient import TestClient + +from nomad.app.main import app + + +@pytest.fixture(scope='session') +def client(): + print('###') + return TestClient(app, base_url='http://testserver/api/v1/') diff --git a/tests/app/v1/routers/__init__.py b/tests/app/v1/routers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/app/routers/common.py b/tests/app/v1/routers/common.py similarity index 100% rename from tests/app/routers/common.py rename to tests/app/v1/routers/common.py diff --git a/tests/app/routers/test_auth.py b/tests/app/v1/routers/test_auth.py similarity index 100% rename from tests/app/routers/test_auth.py rename to tests/app/v1/routers/test_auth.py diff --git a/tests/app/routers/test_datasets.py b/tests/app/v1/routers/test_datasets.py similarity index 100% rename from tests/app/routers/test_datasets.py rename to tests/app/v1/routers/test_datasets.py diff --git a/tests/app/routers/test_entries.py b/tests/app/v1/routers/test_entries.py similarity index 99% rename from tests/app/routers/test_entries.py rename to tests/app/v1/routers/test_entries.py index 492abc68a143927dcad894e25bf42f69eb3315c8..e38feac449315f127c79a0e7d95ef9fc9c190112 100644 --- a/tests/app/routers/test_entries.py +++ b/tests/app/v1/routers/test_entries.py @@ -23,7 +23,7 @@ import io import json from nomad.metainfo.search_extension import search_quantities -from nomad.app.models import AggregateableQuantity, Metric +from nomad.app.v1.models import AggregateableQuantity, Metric from tests.utils import assert_at_least diff --git a/tests/app/routers/test_users.py b/tests/app/v1/routers/test_users.py similarity index 100% rename from tests/app/routers/test_users.py rename to tests/app/v1/routers/test_users.py diff --git a/tests/test_search.py b/tests/test_search.py index 42be98d2499c7ade763b153b366ecff3c7ba9efb..b2bbd1bb44cef3afb88ddebeaa2786d130089144 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -25,7 +25,7 @@ import json from nomad import datamodel, processing, infrastructure, config from nomad.metainfo import search_extension from nomad.search import entry_document, SearchRequest, search, flat -from nomad.app.models import WithQuery +from nomad.app.v1.models import WithQuery def test_init_mapping(elastic):